]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update for PWG1-EVE matching
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
37d21c01 120 fRegisterExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
c6cb3634 124 fJobTag(),
648174cf 125 fOutputSingle(),
5fce53f4 126 fRunPrefix(),
3bdcb562 127 fProofCluster(),
128 fProofDataSet(),
129 fFileForTestMode(),
130 fRootVersionForProof(),
131 fAliRootMode(),
f47d5cb4 132 fMergeDirName(),
4e5c5506 133 fInputFiles(0),
28ce6c9c 134 fPackages(0),
77f34eae 135 fModules(0),
28ce6c9c 136 fProofParam()
c57f56b7 137{
138// Dummy ctor.
139 SetDefaults();
140}
141
142//______________________________________________________________________________
143AliAnalysisAlien::AliAnalysisAlien(const char *name)
144 :AliAnalysisGrid(name),
145 fGridJDL(NULL),
0f389141 146 fMergingJDL(NULL),
c57f56b7 147 fPrice(0),
148 fTTL(0),
149 fSplitMaxInputFileNumber(0),
150 fMaxInitFailed(0),
151 fMasterResubmitThreshold(0),
bb885a9e 152 fNtestFiles(0),
319593fb 153 fNrunsPerMaster(0),
16a4353c 154 fMaxMergeFiles(0),
b3e07543 155 fMaxMergeStages(0),
d2a409b2 156 fNsubmitted(0),
a3e84053 157 fProductionMode(0),
cd11251e 158 fOutputToRunNo(0),
0f389141 159 fMergeViaJDL(0),
160 fFastReadOption(0),
e1c22e21 161 fOverwriteMode(1),
149d288c 162 fNreplicas(2),
3bdcb562 163 fNproofWorkers(0),
27734f0e 164 fNproofWorkersPerSlave(0),
3bdcb562 165 fProofReset(0),
c57f56b7 166 fRunNumbers(),
167 fExecutable(),
0a1c1f7f 168 fExecutableCommand(),
c57f56b7 169 fArguments(),
631c0b05 170 fExecutableArgs(),
c57f56b7 171 fAnalysisMacro(),
172 fAnalysisSource(),
d3b18c4c 173 fValidationScript(),
d5c6455a 174 fAdditionalRootLibs(),
c57f56b7 175 fAdditionalLibs(),
176 fSplitMode(),
177 fAPIVersion(),
178 fROOTVersion(),
179 fAliROOTVersion(),
648174cf 180 fExternalPackages(),
c57f56b7 181 fUser(),
182 fGridWorkingDir(),
183 fGridDataDir(),
184 fDataPattern(),
185 fGridOutputDir(),
186 fOutputArchive(),
187 fOutputFiles(),
188 fInputFormat(),
e7c71df0 189 fDatasetName(),
c57f56b7 190 fJDLName(),
d3b18c4c 191 fTerminateFiles(),
bb885a9e 192 fMergeExcludes(),
37d21c01 193 fRegisterExcludes(),
f965131e 194 fIncludePath(),
bb885a9e 195 fCloseSE(),
0df6ccf2 196 fFriendChainName(),
c6cb3634 197 fJobTag(),
648174cf 198 fOutputSingle(),
5fce53f4 199 fRunPrefix(),
3bdcb562 200 fProofCluster(),
201 fProofDataSet(),
202 fFileForTestMode(),
203 fRootVersionForProof(),
204 fAliRootMode(),
f47d5cb4 205 fMergeDirName(),
4e5c5506 206 fInputFiles(0),
28ce6c9c 207 fPackages(0),
77f34eae 208 fModules(0),
28ce6c9c 209 fProofParam()
c57f56b7 210{
211// Default ctor.
212 SetDefaults();
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
217 :AliAnalysisGrid(other),
218 fGridJDL(NULL),
0f389141 219 fMergingJDL(NULL),
c57f56b7 220 fPrice(other.fPrice),
221 fTTL(other.fTTL),
222 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
223 fMaxInitFailed(other.fMaxInitFailed),
224 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 225 fNtestFiles(other.fNtestFiles),
319593fb 226 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 227 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 228 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 229 fNsubmitted(other.fNsubmitted),
a3e84053 230 fProductionMode(other.fProductionMode),
cd11251e 231 fOutputToRunNo(other.fOutputToRunNo),
0f389141 232 fMergeViaJDL(other.fMergeViaJDL),
233 fFastReadOption(other.fFastReadOption),
234 fOverwriteMode(other.fOverwriteMode),
149d288c 235 fNreplicas(other.fNreplicas),
3bdcb562 236 fNproofWorkers(other.fNproofWorkers),
27734f0e 237 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 238 fProofReset(other.fProofReset),
c57f56b7 239 fRunNumbers(other.fRunNumbers),
240 fExecutable(other.fExecutable),
0a1c1f7f 241 fExecutableCommand(other.fExecutableCommand),
c57f56b7 242 fArguments(other.fArguments),
631c0b05 243 fExecutableArgs(other.fExecutableArgs),
c57f56b7 244 fAnalysisMacro(other.fAnalysisMacro),
245 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 246 fValidationScript(other.fValidationScript),
d5c6455a 247 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 248 fAdditionalLibs(other.fAdditionalLibs),
249 fSplitMode(other.fSplitMode),
250 fAPIVersion(other.fAPIVersion),
251 fROOTVersion(other.fROOTVersion),
252 fAliROOTVersion(other.fAliROOTVersion),
648174cf 253 fExternalPackages(other.fExternalPackages),
c57f56b7 254 fUser(other.fUser),
255 fGridWorkingDir(other.fGridWorkingDir),
256 fGridDataDir(other.fGridDataDir),
257 fDataPattern(other.fDataPattern),
258 fGridOutputDir(other.fGridOutputDir),
259 fOutputArchive(other.fOutputArchive),
260 fOutputFiles(other.fOutputFiles),
261 fInputFormat(other.fInputFormat),
e7c71df0 262 fDatasetName(other.fDatasetName),
c57f56b7 263 fJDLName(other.fJDLName),
d3b18c4c 264 fTerminateFiles(other.fTerminateFiles),
bb885a9e 265 fMergeExcludes(other.fMergeExcludes),
37d21c01 266 fRegisterExcludes(other.fRegisterExcludes),
f965131e 267 fIncludePath(other.fIncludePath),
bb885a9e 268 fCloseSE(other.fCloseSE),
0df6ccf2 269 fFriendChainName(other.fFriendChainName),
c6cb3634 270 fJobTag(other.fJobTag),
648174cf 271 fOutputSingle(other.fOutputSingle),
5fce53f4 272 fRunPrefix(other.fRunPrefix),
3bdcb562 273 fProofCluster(other.fProofCluster),
274 fProofDataSet(other.fProofDataSet),
275 fFileForTestMode(other.fFileForTestMode),
276 fRootVersionForProof(other.fRootVersionForProof),
277 fAliRootMode(other.fAliRootMode),
f47d5cb4 278 fMergeDirName(other.fMergeDirName),
4e5c5506 279 fInputFiles(0),
28ce6c9c 280 fPackages(0),
77f34eae 281 fModules(0),
28ce6c9c 282 fProofParam()
c57f56b7 283{
284// Copy ctor.
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 287 fRunRange[0] = other.fRunRange[0];
288 fRunRange[1] = other.fRunRange[1];
c57f56b7 289 if (other.fInputFiles) {
290 fInputFiles = new TObjArray();
291 TIter next(other.fInputFiles);
292 TObject *obj;
293 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
294 fInputFiles->SetOwner();
295 }
4e5c5506 296 if (other.fPackages) {
297 fPackages = new TObjArray();
298 TIter next(other.fPackages);
299 TObject *obj;
300 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
301 fPackages->SetOwner();
302 }
77f34eae 303 if (other.fModules) {
304 fModules = new TObjArray();
305 fModules->SetOwner();
306 TIter next(other.fModules);
307 AliAnalysisTaskCfg *mod, *crt;
308 while ((crt=(AliAnalysisTaskCfg*)next())) {
309 mod = new AliAnalysisTaskCfg(*crt);
310 fModules->Add(mod);
311 }
312 }
c57f56b7 313}
314
315//______________________________________________________________________________
316AliAnalysisAlien::~AliAnalysisAlien()
317{
318// Destructor.
77f34eae 319 delete fGridJDL;
320 delete fMergingJDL;
321 delete fInputFiles;
322 delete fPackages;
323 delete fModules;
28ce6c9c 324 fProofParam.DeleteAll();
c57f56b7 325}
326
327//______________________________________________________________________________
328AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
329{
330// Assignment.
331 if (this != &other) {
332 AliAnalysisGrid::operator=(other);
333 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 334 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 335 fPrice = other.fPrice;
336 fTTL = other.fTTL;
337 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
338 fMaxInitFailed = other.fMaxInitFailed;
339 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 340 fNtestFiles = other.fNtestFiles;
a3e84053 341 fNrunsPerMaster = other.fNrunsPerMaster;
342 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 343 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 344 fNsubmitted = other.fNsubmitted;
345 fProductionMode = other.fProductionMode;
cd11251e 346 fOutputToRunNo = other.fOutputToRunNo;
0f389141 347 fMergeViaJDL = other.fMergeViaJDL;
348 fFastReadOption = other.fFastReadOption;
349 fOverwriteMode = other.fOverwriteMode;
149d288c 350 fNreplicas = other.fNreplicas;
3bdcb562 351 fNproofWorkers = other.fNproofWorkers;
27734f0e 352 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 353 fProofReset = other.fProofReset;
c57f56b7 354 fRunNumbers = other.fRunNumbers;
355 fExecutable = other.fExecutable;
0a1c1f7f 356 fExecutableCommand = other.fExecutableCommand;
c57f56b7 357 fArguments = other.fArguments;
631c0b05 358 fExecutableArgs = other.fExecutableArgs;
c57f56b7 359 fAnalysisMacro = other.fAnalysisMacro;
360 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 361 fValidationScript = other.fValidationScript;
d5c6455a 362 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 363 fAdditionalLibs = other.fAdditionalLibs;
364 fSplitMode = other.fSplitMode;
365 fAPIVersion = other.fAPIVersion;
366 fROOTVersion = other.fROOTVersion;
367 fAliROOTVersion = other.fAliROOTVersion;
648174cf 368 fExternalPackages = other.fExternalPackages;
c57f56b7 369 fUser = other.fUser;
370 fGridWorkingDir = other.fGridWorkingDir;
371 fGridDataDir = other.fGridDataDir;
372 fDataPattern = other.fDataPattern;
373 fGridOutputDir = other.fGridOutputDir;
374 fOutputArchive = other.fOutputArchive;
375 fOutputFiles = other.fOutputFiles;
376 fInputFormat = other.fInputFormat;
e7c71df0 377 fDatasetName = other.fDatasetName;
c57f56b7 378 fJDLName = other.fJDLName;
d3b18c4c 379 fTerminateFiles = other.fTerminateFiles;
bb885a9e 380 fMergeExcludes = other.fMergeExcludes;
37d21c01 381 fRegisterExcludes = other.fRegisterExcludes;
f965131e 382 fIncludePath = other.fIncludePath;
bb885a9e 383 fCloseSE = other.fCloseSE;
0df6ccf2 384 fFriendChainName = other.fFriendChainName;
c6cb3634 385 fJobTag = other.fJobTag;
648174cf 386 fOutputSingle = other.fOutputSingle;
5fce53f4 387 fRunPrefix = other.fRunPrefix;
3bdcb562 388 fProofCluster = other.fProofCluster;
389 fProofDataSet = other.fProofDataSet;
390 fFileForTestMode = other.fFileForTestMode;
391 fRootVersionForProof = other.fRootVersionForProof;
392 fAliRootMode = other.fAliRootMode;
f47d5cb4 393 fMergeDirName = other.fMergeDirName;
c57f56b7 394 if (other.fInputFiles) {
395 fInputFiles = new TObjArray();
396 TIter next(other.fInputFiles);
397 TObject *obj;
398 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
399 fInputFiles->SetOwner();
400 }
4e5c5506 401 if (other.fPackages) {
402 fPackages = new TObjArray();
403 TIter next(other.fPackages);
404 TObject *obj;
405 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
406 fPackages->SetOwner();
407 }
77f34eae 408 if (other.fModules) {
409 fModules = new TObjArray();
410 fModules->SetOwner();
411 TIter next(other.fModules);
412 AliAnalysisTaskCfg *mod, *crt;
413 while ((crt=(AliAnalysisTaskCfg*)next())) {
414 mod = new AliAnalysisTaskCfg(*crt);
415 fModules->Add(mod);
416 }
417 }
c57f56b7 418 }
419 return *this;
420}
421
37d21c01 422//______________________________________________________________________________
423void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
424{
425// Add a single additional library to be loaded. Extension must be present.
426 TString lib(name);
427 if (!lib.Contains(".")) {
428 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
429 return;
430 }
431 if (fAdditionalLibs.Contains(name)) {
432 Warning("AddAdditionalLibrary", "Library %s already added.", name);
433 return;
434 }
435 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
436 fAdditionalLibs += lib;
437}
438
77f34eae 439//______________________________________________________________________________
440void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
441{
442// Adding a module. Checks if already existing. Becomes owned by this.
443 if (!module) return;
444 if (GetModule(module->GetName())) {
445 Error("AddModule", "A module having the same name %s already added", module->GetName());
446 return;
447 }
448 if (!fModules) {
449 fModules = new TObjArray();
450 fModules->SetOwner();
451 }
452 fModules->Add(module);
453}
454
455//______________________________________________________________________________
456void AliAnalysisAlien::AddModules(TObjArray *list)
457{
458// Adding a list of modules. Checks if already existing. Becomes owned by this.
459 TIter next(list);
460 AliAnalysisTaskCfg *module;
461 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
462}
463
464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckDependencies()
466{
467// Check if all dependencies are satisfied. Reorder modules if needed.
468 Int_t nmodules = GetNmodules();
469 if (!nmodules) {
470 Warning("CheckDependencies", "No modules added yet to check their dependencies");
471 return kTRUE;
472 }
473 AliAnalysisTaskCfg *mod = 0;
474 AliAnalysisTaskCfg *dep = 0;
475 TString depname;
476 Int_t i, j, k;
477 for (i=0; i<nmodules; i++) {
478 mod = (AliAnalysisTaskCfg*) fModules->At(i);
479 Int_t ndeps = mod->GetNdeps();
480 Int_t istart = i;
481 for (j=0; j<ndeps; j++) {
482 depname = mod->GetDependency(j);
483 dep = GetModule(depname);
484 if (!dep) {
485 Error("CheckDependencies","Dependency %s not added for module %s",
486 depname.Data(), mod->GetName());
487 return kFALSE;
488 }
489 if (dep->NeedsDependency(mod->GetName())) {
490 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
491 mod->GetName(), dep->GetName());
492 return kFALSE;
493 }
494 Int_t idep = fModules->IndexOf(dep);
495 // The dependency task must come first
496 if (idep>i) {
497 // Remove at idep and move all objects below up one slot
498 // down to index i included.
499 fModules->RemoveAt(idep);
3e40fd4c 500 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 501 fModules->AddAt(dep, i++);
502 }
503 //Redo from istart if dependencies were inserted
504 if (i>istart) i=istart-1;
505 }
506 }
507 return kTRUE;
508}
509
3e40fd4c 510//______________________________________________________________________________
511AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
512{
513// Create the analysis manager and optionally execute the macro in filename.
514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
515 if (mgr) return mgr;
516 mgr = new AliAnalysisManager(name);
517 mgr->SetGridHandler((AliAnalysisGrid*)this);
518 if (strlen(filename)) {
519 TString line = gSystem->ExpandPathName(filename);
520 line.Prepend(".x ");
521 gROOT->ProcessLine(line.Data());
522 }
523 return mgr;
524}
525
77f34eae 526//______________________________________________________________________________
527Int_t AliAnalysisAlien::GetNmodules() const
528{
529// Get number of modules.
530 if (!fModules) return 0;
531 return fModules->GetEntries();
532}
533
534//______________________________________________________________________________
535AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
536{
537// Get a module by name.
538 if (!fModules) return 0;
539 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
540}
541
542//______________________________________________________________________________
543Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
544{
545// Load a given module.
546 if (mod->IsLoaded()) return kTRUE;
192ddca2 547 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
548 if (!mgr) {
549 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
550 return kFALSE;
551 }
77f34eae 552 Int_t ndeps = mod->GetNdeps();
553 TString depname;
554 for (Int_t j=0; j<ndeps; j++) {
555 depname = mod->GetDependency(j);
556 AliAnalysisTaskCfg *dep = GetModule(depname);
557 if (!dep) {
558 Error("LoadModule","Dependency %s not existing for module %s",
559 depname.Data(), mod->GetName());
560 return kFALSE;
561 }
562 if (!LoadModule(dep)) {
563 Error("LoadModule","Dependency %s for module %s could not be loaded",
564 depname.Data(), mod->GetName());
565 return kFALSE;
566 }
567 }
568 // Load libraries for the module
569 if (!mod->CheckLoadLibraries()) {
570 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
571 return kFALSE;
572 }
192ddca2 573 // Check if a custom file name was requested
574 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
575
576 // Check if a custom terminate file name was requested
577 if (strlen(mod->GetTerminateFileName())) {
578 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
579 fTerminateFiles += mod->GetTerminateFileName();
580 }
581
77f34eae 582 // Execute the macro
583 if (mod->ExecuteMacro()<0) {
584 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
585 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
586 return kFALSE;
587 }
588 // Configure dependencies
589 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
590 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
591 mod->GetConfigMacro()->GetTitle(), mod->GetName());
592 return kFALSE;
593 }
5e8c7c22 594 // Adjust extra libraries
595 Int_t nlibs = mod->GetNlibs();
596 TString lib;
597 for (Int_t i=0; i<nlibs; i++) {
598 lib = mod->GetLibrary(i);
599 if (fAdditionalLibs.Contains(lib)) continue;
600 lib = Form("lib%s.so", lib.Data());
601 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
602 fAdditionalLibs += lib;
603 }
77f34eae 604 return kTRUE;
605}
606
139fe6ba 607//______________________________________________________________________________
608Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
609{
610// Generate the full train.
611 fAdditionalLibs = "";
612 if (!LoadModules()) return kFALSE;
613 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
614 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 615 mgr->RunLocalInit();
139fe6ba 616 mgr->PrintStatus();
617 Int_t productionMode = fProductionMode;
618 SetProductionMode();
619 TString macro = fAnalysisMacro;
620 TString executable = fExecutable;
621 TString validation = fValidationScript;
622 TString execCommand = fExecutableCommand;
623 SetAnalysisMacro(Form("%s.C", name));
624 SetExecutable(Form("%s.sh", name));
625 SetExecutableCommand("aliroot -b -q ");
626 SetValidationScript(Form("%s_validation.sh", name));
627 StartAnalysis();
628 SetProductionMode(productionMode);
629 fAnalysisMacro = macro;
630 fExecutable = executable;
631 fExecutableCommand = execCommand;
632 fValidationScript = validation;
633 return kTRUE;
634}
635
77f34eae 636//______________________________________________________________________________
21ca8e59 637Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 638{
639// Generate test macros for a single module or for the full train.
5e8c7c22 640 fAdditionalLibs = "";
77f34eae 641 if (strlen(modname)) {
21ca8e59 642 if (!CheckDependencies()) return kFALSE;
643 AliAnalysisTaskCfg *mod = GetModule(modname);
644 if (!mod) {
645 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
646 return kFALSE;
647 }
648 if (!LoadModule(mod)) return kFALSE;
649 } else if (!LoadModules()) return kFALSE;
650 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
651 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 652 mgr->RunLocalInit();
21ca8e59 653 mgr->PrintStatus();
654 SetLocalTest(kTRUE);
655 Int_t productionMode = fProductionMode;
656 SetProductionMode();
657 TString macro = fAnalysisMacro;
658 TString executable = fExecutable;
659 TString validation = fValidationScript;
660 TString execCommand = fExecutableCommand;
661 SetAnalysisMacro(Form("%s.C", name));
662 SetExecutable(Form("%s.sh", name));
663 SetExecutableCommand("aliroot -b -q ");
664 SetValidationScript(Form("%s_validation.sh", name));
665 WriteAnalysisFile();
666 WriteAnalysisMacro();
667 WriteExecutable();
668 WriteValidationScript();
669 SetLocalTest(kFALSE);
670 SetProductionMode(productionMode);
671 fAnalysisMacro = macro;
672 fExecutable = executable;
673 fExecutableCommand = execCommand;
674 fValidationScript = validation;
77f34eae 675 return kTRUE;
676}
677
678//______________________________________________________________________________
679Bool_t AliAnalysisAlien::LoadModules()
680{
681// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 682 fAdditionalLibs = "";
21ca8e59 683 Int_t nmodules = GetNmodules();
684 if (!nmodules) {
685 Warning("LoadModules", "No module to be loaded");
686 return kTRUE;
687 }
3e40fd4c 688 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
689 if (!mgr) {
690 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
691 return kFALSE;
692 }
77f34eae 693 if (!CheckDependencies()) return kFALSE;
21ca8e59 694 nmodules = GetNmodules();
77f34eae 695 AliAnalysisTaskCfg *mod;
21ca8e59 696 for (Int_t imod=0; imod<nmodules; imod++) {
697 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 698 if (!LoadModule(mod)) return kFALSE;
699 }
77f34eae 700 return kTRUE;
701}
702
74013535 703//______________________________________________________________________________
704void AliAnalysisAlien::SetRunPrefix(const char *prefix)
705{
706// Set the run number format. Can be a prefix or a format like "%09d"
707 fRunPrefix = prefix;
708 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
709}
710
f965131e 711//______________________________________________________________________________
712void AliAnalysisAlien::AddIncludePath(const char *path)
713{
714// Add include path in the remote analysis macro.
715 TString p(path);
716 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
717 else fIncludePath += Form("-I%s ", path);
718}
719
c57f56b7 720//______________________________________________________________________________
721void AliAnalysisAlien::AddRunNumber(Int_t run)
722{
723// Add a run number to the list of runs to be processed.
724 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 725 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 726}
727
98690895 728//______________________________________________________________________________
729void AliAnalysisAlien::AddRunList(const char* runList)
730{
731// Add several runs into the list of runs; they are expected to be separated by a blank character.
732 TString sList = runList;
733 TObjArray *list = sList.Tokenize(" ");
734 Int_t n = list->GetEntries();
735 for (Int_t i = 0; i < n; i++) {
736 TObjString *os = (TObjString*)list->At(i);
737 AddRunNumber(os->GetString().Atoi());
738 }
739 delete list;
740}
741
ee75cfc3 742//______________________________________________________________________________
743void AliAnalysisAlien::AddRunNumber(const char* run)
744{
745// Add a run number to the list of runs to be processed.
7aa5d539 746 TString runs = run;
747 TObjString *os;
748 TObjArray *arr = runs.Tokenize(" ");
749 TIter next(arr);
750 TString prefix;
751 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
752 while ((os=(TObjString*)next())){
753 if (fRunNumbers.Length()) fRunNumbers += " ";
754 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
755 }
756 delete arr;
ee75cfc3 757}
758
c57f56b7 759//______________________________________________________________________________
760void AliAnalysisAlien::AddDataFile(const char *lfn)
761{
762// Adds a data file to the input to be analysed. The file should be a valid LFN
763// or point to an existing file in the alien workdir.
764 if (!fInputFiles) fInputFiles = new TObjArray();
765 fInputFiles->Add(new TObjString(lfn));
766}
648174cf 767
768//______________________________________________________________________________
769void AliAnalysisAlien::AddExternalPackage(const char *package)
770{
771// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
772 if (fExternalPackages) fExternalPackages += " ";
773 fExternalPackages += package;
774}
775
c57f56b7 776//______________________________________________________________________________
777Bool_t AliAnalysisAlien::Connect()
778{
779// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
780 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 781 if (fProductionMode) return kTRUE;
c57f56b7 782 if (!gGrid) {
783 Info("Connect", "Trying to connect to AliEn ...");
784 TGrid::Connect("alien://");
785 }
786 if (!gGrid || !gGrid->IsConnected()) {
787 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
788 return kFALSE;
789 }
790 fUser = gGrid->GetUser();
791 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
792 return kTRUE;
793}
794
795//______________________________________________________________________________
796void AliAnalysisAlien::CdWork()
797{
798// Check validity of alien workspace. Create directory if possible.
799 if (!Connect()) {
800 Error("CdWork", "Alien connection required");
801 return;
802 }
803 TString homedir = gGrid->GetHomeDirectory();
804 TString workdir = homedir + fGridWorkingDir;
923e2ca5 805 if (DirectoryExists(workdir)) {
806 gGrid->Cd(workdir);
807 return;
808 }
809 // Work directory not existing - create it
810 gGrid->Cd(homedir);
b93f8109 811 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 812 gGrid->Cd(fGridWorkingDir);
d3b18c4c 813 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 814 } else {
d3b18c4c 815 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 816 workdir.Data(), homedir.Data());
817 fGridWorkingDir = "";
818 }
c57f56b7 819}
820
348be253 821//______________________________________________________________________________
822Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
823{
824// Check if file copying is possible.
d3b18c4c 825 if (fProductionMode) return kTRUE;
b5ae4fe0 826 TString salienpath(alienpath);
827 if (salienpath.Contains(" ")) {
828 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
829 return kFALSE;
830 }
348be253 831 if (!Connect()) {
832 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
833 return kFALSE;
834 }
d3339be3 835 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
836 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 837 // Check if alien_CLOSE_SE is defined
838 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
839 if (!closeSE.IsNull()) {
840 Info("CheckFileCopy", "Your current close storage is pointing to: \
841 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
842 } else {
843 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
844 }
845 // Check if grid directory exists.
846 if (!DirectoryExists(alienpath)) {
847 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
848 return kFALSE;
849 }
b5ae4fe0 850 TString stest = "plugin_test_copy";
851 TFile f(stest, "RECREATE");
348be253 852 // User may not have write permissions to current directory
853 if (f.IsZombie()) {
854 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
855 gSystem->WorkingDirectory());
856 return kFALSE;
857 }
858 f.Close();
b5ae4fe0 859 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
860 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 861 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 862 \n# 1. Make sure you have write permissions there. If this is the case: \
863 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
864 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
865 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
866 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 867 gSystem->Unlink(stest.Data());
348be253 868 return kFALSE;
869 }
b5ae4fe0 870 gSystem->Unlink(stest.Data());
871 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 872 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 873 return kTRUE;
874}
875
c57f56b7 876//______________________________________________________________________________
877Bool_t AliAnalysisAlien::CheckInputData()
878{
879// Check validity of input data. If necessary, create xml files.
d3b18c4c 880 if (fProductionMode) return kTRUE;
a8739e8a 881 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 882 if (!fGridDataDir.Length()) {
883 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
884 return kFALSE;
885 }
9f020952 886 if (fMergeViaJDL) {
887 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
888 return kFALSE;
889 }
d2a409b2 890 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 891 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
892 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 893 return kTRUE;
c57f56b7 894 }
895 // Process declared files
a2f5fc01 896 Bool_t isCollection = kFALSE;
897 Bool_t isXml = kFALSE;
898 Bool_t useTags = kFALSE;
c57f56b7 899 Bool_t checked = kFALSE;
d3b18c4c 900 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 901 TString file;
902 TString workdir = gGrid->GetHomeDirectory();
903 workdir += fGridWorkingDir;
904 if (fInputFiles) {
905 TObjString *objstr;
906 TIter next(fInputFiles);
907 while ((objstr=(TObjString*)next())) {
908 file = workdir;
909 file += "/";
910 file += objstr->GetString();
911 // Store full lfn path
912 if (FileExists(file)) objstr->SetString(file);
913 else {
914 file = objstr->GetName();
915 if (!FileExists(objstr->GetName())) {
916 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
917 objstr->GetName(), workdir.Data());
918 return kFALSE;
919 }
920 }
921 Bool_t iscoll, isxml, usetags;
922 CheckDataType(file, iscoll, isxml, usetags);
923 if (!checked) {
924 checked = kTRUE;
a2f5fc01 925 isCollection = iscoll;
926 isXml = isxml;
927 useTags = usetags;
928 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 929 } else {
a2f5fc01 930 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 931 Error("CheckInputData", "Some conflict was found in the types of inputs");
932 return kFALSE;
933 }
934 }
935 }
936 }
937 // Process requested run numbers
a8739e8a 938 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 939 // Check validity of alien data directory
940 if (!fGridDataDir.Length()) {
941 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
942 return kFALSE;
943 }
923e2ca5 944 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 945 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
946 return kFALSE;
947 }
a2f5fc01 948 if (isCollection) {
c57f56b7 949 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
950 return kFALSE;
951 }
952
a2f5fc01 953 if (checked && !isXml) {
c57f56b7 954 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
955 return kFALSE;
956 }
957 // Check validity of run number(s)
958 TObjArray *arr;
959 TObjString *os;
74013535 960 TString format;
319593fb 961 Int_t nruns = 0;
904f9f5f 962 TString schunk, schunk2;
c57f56b7 963 TString path;
964 if (!checked) {
965 checked = kTRUE;
a2f5fc01 966 useTags = fDataPattern.Contains("tag");
967 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 968 }
a2f5fc01 969 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 970 Error("CheckInputData", "Cannot mix input files using/not using tags");
971 return kFALSE;
972 }
973 if (fRunNumbers.Length()) {
a8739e8a 974 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 975 arr = fRunNumbers.Tokenize(" ");
976 TIter next(arr);
977 while ((os=(TObjString*)next())) {
978 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 979 if (!DirectoryExists(path)) {
980 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 981 continue;
c57f56b7 982 }
983 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
984 TString msg = "\n##### file: ";
985 msg += path;
986 msg += " type: xml_collection;";
a2f5fc01 987 if (useTags) msg += " using_tags: Yes";
c57f56b7 988 else msg += " using_tags: No";
23329835 989 Info("CheckDataType", "%s", msg.Data());
319593fb 990 if (fNrunsPerMaster<2) {
d2a409b2 991 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 992 } else {
993 nruns++;
994 if (((nruns-1)%fNrunsPerMaster) == 0) {
995 schunk = os->GetString();
996 }
997 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
998 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 999 AddDataFile(schunk);
319593fb 1000 }
c57f56b7 1001 }
1002 delete arr;
a8739e8a 1003 } else {
1004 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1005 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1006 format = Form("%%s/%s ", fRunPrefix.Data());
1007 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1008 if (!DirectoryExists(path)) {
a8739e8a 1009 continue;
1010 }
74013535 1011 format = Form("%%s/%s.xml", fRunPrefix.Data());
1012 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1013 TString msg = "\n##### file: ";
1014 msg += path;
1015 msg += " type: xml_collection;";
a2f5fc01 1016 if (useTags) msg += " using_tags: Yes";
a8739e8a 1017 else msg += " using_tags: No";
23329835 1018 Info("CheckDataType", "%s", msg.Data());
319593fb 1019 if (fNrunsPerMaster<2) {
74013535 1020 format = Form("%s.xml", fRunPrefix.Data());
1021 AddDataFile(Form(format.Data(),irun));
319593fb 1022 } else {
1023 nruns++;
1024 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1025 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1026 }
74013535 1027 format = Form("_%s.xml", fRunPrefix.Data());
1028 schunk2 = Form(format.Data(), irun);
319593fb 1029 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1030 schunk += schunk2;
d2a409b2 1031 AddDataFile(schunk);
319593fb 1032 }
a8739e8a 1033 }
904f9f5f 1034 if (!fInputFiles) {
1035 schunk += schunk2;
1036 AddDataFile(schunk);
1037 }
c57f56b7 1038 }
1039 return kTRUE;
1040}
1041
21ca8e59 1042//______________________________________________________________________________
43b807f6 1043Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1044{
1045// Copy data from the given grid directory according a pattern and make a local
1046// dataset.
2ba4c287 1047// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1048 if (!Connect()) {
1049 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1050 return kFALSE;
1051 }
1052 if (!DirectoryExists(griddir)) {
1053 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1054 return kFALSE;
1055 }
1056 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1057 printf("Running command: %s\n", command.Data());
1058 TGridResult *res = gGrid->Command(command);
1059 Int_t nfound = res->GetEntries();
1060 if (!nfound) {
1061 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1062 return kFALSE;
1063 }
1064 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1065
1066 // archives
1067 TObjArray* additionalArchives = 0;
1068 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1069 additionalArchives = TString(archivefile).Tokenize(";");
1070 archivefile = additionalArchives->At(0)->GetName();
1071 additionalArchives->RemoveAt(0);
1072 additionalArchives->Compress();
1073 }
1074
21ca8e59 1075 // Copy files locally
1076 ofstream out;
1077 out.open(output, ios::out);
1078 TMap *map;
1079 TString turl, dirname, filename, temp;
1080 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1081 gSystem->MakeDirectory(outputdir);
1082 gSystem->ChangeDirectory(outputdir);
21ca8e59 1083 for (Int_t i=0; i<nfound; i++) {
1084 map = (TMap*)res->At(i);
1085 turl = map->GetValue("turl")->GetName();
1086 filename = gSystem->BaseName(turl.Data());
1087 dirname = gSystem->DirName(turl.Data());
1088 dirname = gSystem->BaseName(dirname.Data());
1089 gSystem->MakeDirectory(dirname);
43b807f6 1090
1091 TString source(turl);
1092 TString targetFileName(filename);
1093
1094 if (strlen(archivefile) > 0) {
1095 // TODO here the archive in which the file resides should be determined
1096 // however whereis returns only a guid, and guid2lfn does not work
1097 // Therefore we use the one provided as argument for now
1098 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1099 targetFileName = archivefile;
1100 }
1101 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
2ba4c287 1102 Bool_t success = kTRUE;
1103 if (additionalArchives)
1104 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++)
1105 {
1106 TString target;
1107 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1108 gSystem->MakeDirectory(gSystem->DirName(target));
1109 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1110 }
1111
1112 if (success) {
1113 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1114 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1115 }
21ca8e59 1116 }
1117 }
1118 gSystem->ChangeDirectory(cdir);
1119 delete res;
2ba4c287 1120 if (additionalArchives)
1121 delete additionalArchives;
21ca8e59 1122 return kTRUE;
1123}
1124
c57f56b7 1125//______________________________________________________________________________
1126Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1127{
1128// Create dataset for the grid data directory + run number.
3af861ff 1129 const Int_t gMaxEntries = 15000;
d3b18c4c 1130 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1131 if (!Connect()) {
1132 Error("CreateDataset", "Cannot create dataset with no grid connection");
1133 return kFALSE;
1134 }
1135
1136 // Cd workspace
d3b18c4c 1137 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1138 TString workdir = gGrid->GetHomeDirectory();
1139 workdir += fGridWorkingDir;
1140
1141 // Compose the 'find' command arguments
74013535 1142 TString format;
c57f56b7 1143 TString command;
1144 TString options = "-x collection ";
bb885a9e 1145 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1146 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1147 TString conditions = "";
3af861ff 1148 Int_t nstart = 0;
1149 Int_t ncount = 0;
1150 Int_t stage = 0;
c57f56b7 1151 TString file;
1152 TString path;
319593fb 1153 Int_t nruns = 0;
904f9f5f 1154 TString schunk, schunk2;
ab254fd1 1155 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1156 if (!fRunNumbers.Length() && !fRunRange[0]) {
1157 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1158 // Make a single data collection from data directory.
1159 path = fGridDataDir;
923e2ca5 1160 if (!DirectoryExists(path)) {
d2a409b2 1161 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1162 return kFALSE;
1163 }
923e2ca5 1164// CdWork();
d2a409b2 1165 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1166 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1167 while (1) {
1168 ncount = 0;
1169 stage++;
1170 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1171 command = "find ";
1172 command += Form("%s -o %d ",options.Data(), nstart);
1173 command += path;
1174 command += " ";
1175 command += pattern;
1176 command += conditions;
1177 printf("command: %s\n", command.Data());
1178 TGridResult *res = gGrid->Command(command);
1179 if (res) delete res;
1180 // Write standard output to file
1181 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1182 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1183 Bool_t nullFile = kFALSE;
1184 if (!hasGrep) {
1185 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1186 } else {
1187 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1188 if (nullFile) {
1189 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1190 gSystem->Exec("rm -f __tmp*");
1191 return kFALSE;
1192 }
1193 TString line;
1194 ifstream in;
1195 in.open("__tmp__");
1196 in >> line;
1197 in.close();
1198 gSystem->Exec("rm -f __tmp__");
1199 ncount = line.Atoi();
1200 }
1201 }
1202 if (ncount == gMaxEntries) {
1203 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1204 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1205 if (!cbase) cbase = cadd;
1206 else {
1207 cbase->Add(cadd);
1208 delete cadd;
defd7a3a 1209 }
3af861ff 1210 nstart += ncount;
1211 } else {
1212 if (cbase) {
1213 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1214 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1215 cbase->Add(cadd);
1216 delete cadd;
1217 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1218 delete cbase; cbase = 0;
1219 } else {
1220 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1221 }
1222 gSystem->Exec("rm -f __tmp*");
1223 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1224 break;
1225 }
0f389141 1226 }
1227 Bool_t fileExists = FileExists(file);
1228 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1229 // Copy xml file to alien space
0f389141 1230 if (fileExists) gGrid->Rm(file);
d2a409b2 1231 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1232 if (!FileExists(file)) {
1233 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1234 return kFALSE;
1235 }
1236 // Update list of files to be processed.
1237 }
1238 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1239 return kTRUE;
1240 }
c57f56b7 1241 // Several runs
a2f5fc01 1242 Bool_t nullResult = kTRUE;
a8739e8a 1243 if (fRunNumbers.Length()) {
1244 TObjArray *arr = fRunNumbers.Tokenize(" ");
1245 TObjString *os;
1246 TIter next(arr);
1247 while ((os=(TObjString*)next())) {
3af861ff 1248 nstart = 0;
1249 stage = 0;
dc030a24 1250 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1251 if (!DirectoryExists(path)) continue;
1252// CdWork();
a8739e8a 1253 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1254 else file = Form("%s.xml", os->GetString().Data());
319593fb 1255 // If local collection file does not exist, create it via 'find' command.
3af861ff 1256 while (1) {
1257 ncount = 0;
1258 stage++;
1259 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1260 command = "find ";
1261 command += Form("%s -o %d ",options.Data(), nstart);
1262 command += path;
1263 command += pattern;
1264 command += conditions;
1265 TGridResult *res = gGrid->Command(command);
1266 if (res) delete res;
1267 // Write standard output to file
1268 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1269 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1270 Bool_t nullFile = kFALSE;
1271 if (!hasGrep) {
1272 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1273 } else {
1274 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1275 if (nullFile) {
1276 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1277 gSystem->Exec("rm -f __tmp*");
1278 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1279 break;
3af861ff 1280 }
1281 TString line;
1282 ifstream in;
1283 in.open("__tmp__");
1284 in >> line;
1285 in.close();
1286 gSystem->Exec("rm -f __tmp__");
1287 ncount = line.Atoi();
1288 }
1289 nullResult = kFALSE;
1290 }
1291 if (ncount == gMaxEntries) {
1292 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1293 if (fNrunsPerMaster > 1) {
1294 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1295 file.Data(),gMaxEntries);
1296 return kFALSE;
1297 }
1298 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1299 if (!cbase) cbase = cadd;
1300 else {
1301 cbase->Add(cadd);
1302 delete cadd;
defd7a3a 1303 }
3af861ff 1304 nstart += ncount;
1305 } else {
c89a84bc 1306 if (cbase && fNrunsPerMaster<2) {
3af861ff 1307 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1308 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1309 cbase->Add(cadd);
1310 delete cadd;
1311 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1312 delete cbase; cbase = 0;
1313 } else {
1314 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1315 }
1316 gSystem->Exec("rm -f __tmp*");
1317 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1318 break;
d3339be3 1319 }
3af861ff 1320 }
a8739e8a 1321 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1322 // Check if there is one run per master job.
1323 if (fNrunsPerMaster<2) {
1324 if (FileExists(file)) {
0f389141 1325 if (fOverwriteMode) gGrid->Rm(file);
1326 else {
1327 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1328 continue;
1329 }
319593fb 1330 }
1331 // Copy xml file to alien space
1332 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1333 if (!FileExists(file)) {
1334 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1335 delete arr;
1336 return kFALSE;
1337 }
1338 } else {
1339 nruns++;
1340 if (((nruns-1)%fNrunsPerMaster) == 0) {
1341 schunk = os->GetString();
1342 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1343 } else {
1344 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1345 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1346 cbase->Add(cadd);
1347 delete cadd;
1348 }
1349 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1350 continue;
1351 }
1352 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1353 if (FileExists(schunk)) {
1354 if (fOverwriteMode) gGrid->Rm(file);
1355 else {
1356 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1357 continue;
1358 }
319593fb 1359 }
84fcd93f 1360 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1361 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1362 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1363 if (!FileExists(schunk)) {
1364 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1365 delete arr;
1366 return kFALSE;
1367 }
d3339be3 1368 }
a8739e8a 1369 }
1370 delete arr;
a2f5fc01 1371 if (nullResult) {
d3339be3 1372 Error("CreateDataset", "No valid dataset corresponding to the query!");
1373 return kFALSE;
1374 }
a8739e8a 1375 } else {
1376 // Process a full run range.
1377 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1378 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1379 nstart = 0;
1380 stage = 0;
74013535 1381 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1382 if (!DirectoryExists(path)) continue;
1383// CdWork();
74013535 1384 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1385 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1386 else file = Form(format.Data(), irun);
0f389141 1387 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1388 if (fOverwriteMode) gGrid->Rm(file);
1389 else {
1390 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1391 continue;
1392 }
a8739e8a 1393 }
319593fb 1394 // If local collection file does not exist, create it via 'find' command.
3af861ff 1395 while (1) {
1396 ncount = 0;
1397 stage++;
1398 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1399 command = "find ";
1400 command += Form("%s -o %d ",options.Data(), nstart);
1401 command += path;
1402 command += pattern;
1403 command += conditions;
1404 TGridResult *res = gGrid->Command(command);
1405 if (res) delete res;
1406 // Write standard output to file
1407 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1408 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1409 Bool_t nullFile = kFALSE;
1410 if (!hasGrep) {
1411 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1412 } else {
1413 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1414 if (nullFile) {
1415 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1416 gSystem->Exec("rm -f __tmp*");
1604c9ed 1417 break;
3af861ff 1418 }
1419 TString line;
1420 ifstream in;
1421 in.open("__tmp__");
1422 in >> line;
1423 in.close();
1424 gSystem->Exec("rm -f __tmp__");
1425 ncount = line.Atoi();
1426 }
1427 nullResult = kFALSE;
1428 }
1429 if (ncount == gMaxEntries) {
1430 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1431 if (fNrunsPerMaster > 1) {
1432 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1433 file.Data(),gMaxEntries);
1434 return kFALSE;
1435 }
1436 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1437 if (!cbase) cbase = cadd;
1438 else {
1439 cbase->Add(cadd);
1440 delete cadd;
defd7a3a 1441 }
3af861ff 1442 nstart += ncount;
1443 } else {
c89a84bc 1444 if (cbase && fNrunsPerMaster<2) {
3af861ff 1445 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1446 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1447 cbase->Add(cadd);
1448 delete cadd;
1449 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1450 delete cbase; cbase = 0;
1451 } else {
1452 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1453 }
1454 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1455 break;
d3339be3 1456 }
319593fb 1457 }
a8739e8a 1458 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1459 // Check if there is one run per master job.
1460 if (fNrunsPerMaster<2) {
1461 if (FileExists(file)) {
0f389141 1462 if (fOverwriteMode) gGrid->Rm(file);
1463 else {
1464 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1465 continue;
1466 }
319593fb 1467 }
1468 // Copy xml file to alien space
1469 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1470 if (!FileExists(file)) {
1471 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1472 return kFALSE;
1473 }
1474 } else {
1475 nruns++;
95e5b448 1476 // Check if the collection for the chunk exist locally.
1477 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1478 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1479 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1480 else continue;
1481 }
84fcd93f 1482 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1483 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1484 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1485 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1486 } else {
1487 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1488 cbase->Add(cadd);
1489 delete cadd;
1490 }
74013535 1491 format = Form("%%s_%s.xml", fRunPrefix.Data());
1492 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1493 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1494 continue;
1495 }
904f9f5f 1496 schunk = schunk2;
319593fb 1497 if (FileExists(schunk)) {
0f389141 1498 if (fOverwriteMode) gGrid->Rm(schunk);
1499 else {
1500 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1501 continue;
1502 }
319593fb 1503 }
84fcd93f 1504 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1505 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1506 if (FileExists(schunk)) {
0f389141 1507 if (fOverwriteMode) gGrid->Rm(schunk);
1508 else {
1509 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1510 continue;
1511 }
95e5b448 1512 }
319593fb 1513 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1514 if (!FileExists(schunk)) {
1515 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1516 return kFALSE;
1517 }
1518 }
c57f56b7 1519 }
a2f5fc01 1520 if (nullResult) {
d3339be3 1521 Error("CreateDataset", "No valid dataset corresponding to the query!");
1522 return kFALSE;
1523 }
a8739e8a 1524 }
c57f56b7 1525 return kTRUE;
1526}
1527
1528//______________________________________________________________________________
1529Bool_t AliAnalysisAlien::CreateJDL()
1530{
1531// Generate a JDL file according to current settings. The name of the file is
1532// specified by fJDLName.
1533 Bool_t error = kFALSE;
1534 TObjArray *arr = 0;
1535 Bool_t copy = kTRUE;
d3b18c4c 1536 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1537 Bool_t generate = kTRUE;
1538 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1539 if (!Connect()) {
1540 Error("CreateJDL", "Alien connection required");
1541 return kFALSE;
1542 }
1543 // Check validity of alien workspace
d3b18c4c 1544 TString workdir;
1545 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1546 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1547 workdir += fGridWorkingDir;
1548 if (generate) {
1549 TObjString *os;
1550 if (!fInputFiles) {
1551 Error("CreateJDL()", "Define some input files for your analysis.");
1552 error = kTRUE;
1553 }
1554 // Compose list of input files
1555 // Check if output files were defined
1556 if (!fOutputFiles.Length()) {
1557 Error("CreateJDL", "You must define at least one output file");
1558 error = kTRUE;
1559 }
1560 // Check if an output directory was defined and valid
1561 if (!fGridOutputDir.Length()) {
1562 Error("CreateJDL", "You must define AliEn output directory");
1563 error = kTRUE;
1564 } else {
d3b18c4c 1565 if (!fProductionMode) {
1566 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1567 if (!DirectoryExists(fGridOutputDir)) {
1568 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1569 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1570 } else {
1571 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1572 // error = kTRUE;
1573 }
67a3b254 1574 } else {
1575 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1576 }
1577 gGrid->Cd(workdir);
1578 }
c57f56b7 1579 }
1580 // Exit if any error up to now
1581 if (error) return kFALSE;
1582 // Set JDL fields
0f389141 1583 if (!fUser.IsNull()) {
1584 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1585 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1586 }
1587 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1588 TString mergeExec = fExecutable;
1589 mergeExec.ReplaceAll(".sh", "_merge.sh");
1590 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1591 mergeExec.ReplaceAll(".sh", ".C");
1592 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1593 if (!fArguments.IsNull())
1594 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1595 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1596 else {
1597 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1598 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1599 }
b3e07543 1600
1f0d1ca2 1601 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1602 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1603 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1604 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1605
0f389141 1606 if (fMaxInitFailed > 0) {
c57f56b7 1607 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1608 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1609 }
1610 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1611 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1612 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1613 }
1614 if (!IsOneStageMerging()) {
f47d5cb4 1615 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1616 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1617 }
1618 if (fSplitMode.Length()) {
c57f56b7 1619 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1620 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1621 }
1622 fMergingJDL->SetValue("Split", "\"se\"");
1623 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1624 if (!fAliROOTVersion.IsNull()) {
1625 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1626 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1627 }
1628 if (!fROOTVersion.IsNull()) {
c57f56b7 1629 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1630 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1631 }
1632 if (!fAPIVersion.IsNull()) {
c57f56b7 1633 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1634 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1635 }
648174cf 1636 if (!fExternalPackages.IsNull()) {
1637 arr = fExternalPackages.Tokenize(" ");
1638 TIter next(arr);
1639 while ((os=(TObjString*)next())) {
1640 TString pkgname = os->GetString();
1641 Int_t index = pkgname.Index("::");
1642 TString pkgversion = pkgname(index+2, pkgname.Length());
1643 pkgname.Remove(index);
1644 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1645 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1646 }
1647 delete arr;
1648 }
0f389141 1649 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1650 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1651 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1652 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1653 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1654 TString analysisFile = fExecutable;
1655 analysisFile.ReplaceAll(".sh", ".root");
1656 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1657 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1658 if (fAdditionalLibs.Length()) {
1659 arr = fAdditionalLibs.Tokenize(" ");
1660 TIter next(arr);
1661 while ((os=(TObjString*)next())) {
1662 if (os->GetString().Contains(".so")) continue;
1663 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1664 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1665 }
1666 delete arr;
1667 }
4e5c5506 1668 if (fPackages) {
1669 TIter next(fPackages);
1670 TObject *obj;
0f389141 1671 while ((obj=next())) {
4e5c5506 1672 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1673 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1674 }
4e5c5506 1675 }
c57f56b7 1676 if (fOutputArchive.Length()) {
1677 arr = fOutputArchive.Tokenize(" ");
1678 TIter next(arr);
0f389141 1679 Bool_t first = kTRUE;
1680 const char *comment = "Files to be archived";
1681 const char *comment1 = comment;
1682 while ((os=(TObjString*)next())) {
1683 if (!first) comment = NULL;
1684 if (!os->GetString().Contains("@") && fCloseSE.Length())
1685 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1686 else
1687 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1688 first = kFALSE;
1689 }
c57f56b7 1690 delete arr;
f790bc1b 1691 // Output archive for the merging jdl
1692 TString outputArchive;
1693 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1694 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1695 // Add normal output files, extra files + terminate files
1696 TString files = GetListOfFiles("outextter");
37d21c01 1697 // Do not register files in fRegisterExcludes
1698 if (!fRegisterExcludes.IsNull()) {
1699 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1700 TIter next1(arr);
1701 while ((os=(TObjString*)next1())) {
1702 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1703 files.ReplaceAll(os->GetString(),"");
1704 }
66d71516 1705 delete arr;
0f389141 1706 }
f790bc1b 1707 files.ReplaceAll(".root", "*.root");
b83f84f0 1708 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1709 } else {
95f84779 1710 TString files = fOutputArchive;
1711 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1712 outputArchive = files;
f790bc1b 1713 }
0f389141 1714 arr = outputArchive.Tokenize(" ");
705adb3e 1715 TIter next2(arr);
0f389141 1716 comment = comment1;
1717 first = kTRUE;
705adb3e 1718 while ((os=(TObjString*)next2())) {
0f389141 1719 if (!first) comment = NULL;
0cdf65a8 1720 TString currentfile = os->GetString();
0cdf65a8 1721 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1722 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1723 else
0cdf65a8 1724 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1725 first = kFALSE;
1726 }
1727 delete arr;
c57f56b7 1728 }
149d288c 1729 arr = fOutputFiles.Tokenize(",");
c57f56b7 1730 TIter next(arr);
0f389141 1731 Bool_t first = kTRUE;
f790bc1b 1732 const char *comment = "Files to be saved";
43da816a 1733 while ((os=(TObjString*)next())) {
e1eaf596 1734 // Ignore ouputs in jdl that are also in outputarchive
1735 TString sout = os->GetString();
f790bc1b 1736 sout.ReplaceAll("*", "");
1737 sout.ReplaceAll(".root", "");
e1eaf596 1738 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1739 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1740 // Ignore fRegisterExcludes
1741 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1742 if (!first) comment = NULL;
43da816a 1743 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1744 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1745 else
0f389141 1746 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1747 first = kFALSE;
f790bc1b 1748 if (fMergeExcludes.Contains(sout)) continue;
1749 if (!os->GetString().Contains("@") && fCloseSE.Length())
1750 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1751 else
1752 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1753 }
c57f56b7 1754 delete arr;
0f389141 1755 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1756 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1757 TString validationScript = fValidationScript;
0f389141 1758 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1759 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1760 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1761 if (fMasterResubmitThreshold) {
1762 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1763 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1764 }
d2a409b2 1765 // Write a jdl with 2 input parameters: collection name and output dir name.
1766 WriteJDL(copy);
c57f56b7 1767 }
1768 // Copy jdl to grid workspace
a8739e8a 1769 if (copy) {
b5e4aaa7 1770 // Check if an output directory was defined and valid
1771 if (!fGridOutputDir.Length()) {
1772 Error("CreateJDL", "You must define AliEn output directory");
1773 return kFALSE;
1774 } else {
1775 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1776 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1777 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1778 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1779 } else {
1780 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1781 return kFALSE;
1782 }
1783 }
1784 gGrid->Cd(workdir);
1785 }
648174cf 1786 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1787 TString mergeJDLName = fExecutable;
1788 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1789 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1790 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1791 if (fProductionMode) {
648174cf 1792 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1793 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1794 }
648174cf 1795 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1796 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1797 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1798 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1799 Fatal("","Terminating");
1800// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1801 if (fMergeViaJDL) {
1802 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1803// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1804 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1805 Fatal("","Terminating");
0f389141 1806 }
648174cf 1807 }
c57f56b7 1808 if (fAdditionalLibs.Length()) {
e7c71df0 1809 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1810 TObjString *os;
1811 TIter next(arr);
1812 while ((os=(TObjString*)next())) {
c57f56b7 1813 if (os->GetString().Contains(".so")) continue;
4e5c5506 1814 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1815 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1816// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1817 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1818 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1819 Fatal("","Terminating");
c57f56b7 1820 }
1821 delete arr;
1822 }
4e5c5506 1823 if (fPackages) {
1824 TIter next(fPackages);
1825 TObject *obj;
1826 while ((obj=next())) {
fdbbc7be 1827 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1828 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1829// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1830 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1831 Form("%s/%s", workdir.Data(), obj->GetName())))
1832 Fatal("","Terminating");
4e5c5506 1833 }
1834 }
c57f56b7 1835 }
1836 return kTRUE;
1837}
1838
a8739e8a 1839//______________________________________________________________________________
d2a409b2 1840Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1841{
1842// Writes one or more JDL's corresponding to findex. If findex is negative,
1843// all run numbers are considered in one go (jdl). For non-negative indices
1844// they correspond to the indices in the array fInputFiles.
1845 if (!fInputFiles) return kFALSE;
d3b18c4c 1846 TObject *os;
1847 TString workdir;
1848 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1849 workdir += fGridWorkingDir;
f47d5cb4 1850 TString stageName = "$2";
1851 if (fProductionMode) stageName = "$4";
1852 if (!fMergeDirName.IsNull()) {
1853 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1854 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1855 } else {
1856 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1857 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1858 }
d3b18c4c 1859 if (fProductionMode) {
a8739e8a 1860 TIter next(fInputFiles);
b3e07543 1861 while ((os=next())) {
d3b18c4c 1862 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1863 }
56754bf6 1864 if (!fOutputToRunNo)
1865 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1866 else
1867 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1868 } else {
1869 if (!fRunNumbers.Length() && !fRunRange[0]) {
1870 // One jdl with no parameters in case input data is specified by name.
1871 TIter next(fInputFiles);
1872 while ((os=next()))
1873 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1874 if (!fOutputSingle.IsNull())
1875 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1876 else {
1877 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1878 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1879 }
1880 } else {
1881 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1882 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1883 if (!fOutputSingle.IsNull()) {
1884 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1885 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1886 } else {
1887 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1888 }
1889 }
a8739e8a 1890 }
1891
a8739e8a 1892 // Generate the JDL as a string
1893 TString sjdl = fGridJDL->Generate();
0f389141 1894 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1895 // Final merge jdl
1896 if (!fMergeDirName.IsNull()) {
1897 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1898 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1899 } else {
1900 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1901 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1902 }
b3e07543 1903 TString sjdl2 = fMergingJDL->Generate();
1904 Int_t index, index1;
a8739e8a 1905 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1906 sjdl.ReplaceAll("(member", "\n (member");
1907 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1908 sjdl.ReplaceAll("{", "{\n ");
1909 sjdl.ReplaceAll("};", "\n};");
1910 sjdl.ReplaceAll("{\n \n", "{\n");
1911 sjdl.ReplaceAll("\n\n", "\n");
1912 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1913 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1914 sjdl1.ReplaceAll("(member", "\n (member");
1915 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1916 sjdl1.ReplaceAll("{", "{\n ");
1917 sjdl1.ReplaceAll("};", "\n};");
1918 sjdl1.ReplaceAll("{\n \n", "{\n");
1919 sjdl1.ReplaceAll("\n\n", "\n");
1920 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1921 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1922 sjdl2.ReplaceAll("(member", "\n (member");
1923 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1924 sjdl2.ReplaceAll("{", "{\n ");
1925 sjdl2.ReplaceAll("};", "\n};");
1926 sjdl2.ReplaceAll("{\n \n", "{\n");
1927 sjdl2.ReplaceAll("\n\n", "\n");
1928 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1929 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1930 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1931 index = sjdl.Index("JDLVariables");
1932 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1933 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1934 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1935 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1936 index = fJobTag.Index(":");
1937 if (index < 0) index = fJobTag.Length();
1938 TString jobTag = fJobTag;
f47d5cb4 1939 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1940 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1941 if (fProductionMode) {
1942 sjdl1.Prepend("# Generated merging jdl (production mode) \
1943 \n# $1 = full alien path to output directory to be merged \
1944 \n# $2 = train number \
1945 \n# $3 = production (like LHC10b) \
1946 \n# $4 = merging stage \
1947 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1948 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1949 sjdl2.Prepend("# Generated merging jdl \
1950 \n# $1 = full alien path to output directory to be merged \
1951 \n# $2 = train number \
1952 \n# $3 = production (like LHC10b) \
1953 \n# $4 = merging stage \
1954 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1955 } else {
1956 sjdl1.Prepend("# Generated merging jdl \
1957 \n# $1 = full alien path to output directory to be merged \
1958 \n# $2 = merging stage \
03335324 1959 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1960 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1961 sjdl2.Prepend("# Generated merging jdl \
1962 \n# $1 = full alien path to output directory to be merged \
1963 \n# $2 = merging stage \
03335324 1964 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1965 }
0f389141 1966 index = sjdl1.Index("JDLVariables");
1967 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1968 index = sjdl2.Index("JDLVariables");
1969 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1970 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1971 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1972 index = sjdl2.Index("Split =");
1973 if (index>=0) {
1974 index1 = sjdl2.Index("\n", index);
1975 sjdl2.Remove(index, index1-index+1);
1976 }
1977 index = sjdl2.Index("SplitMaxInputFileNumber");
1978 if (index>=0) {
1979 index1 = sjdl2.Index("\n", index);
1980 sjdl2.Remove(index, index1-index+1);
1981 }
1982 index = sjdl2.Index("InputDataCollection");
1983 if (index>=0) {
1984 index1 = sjdl2.Index(";", index);
1985 sjdl2.Remove(index, index1-index+1);
1986 }
1987 index = sjdl2.Index("InputDataListFormat");
1988 if (index>=0) {
1989 index1 = sjdl2.Index("\n", index);
1990 sjdl2.Remove(index, index1-index+1);
1991 }
1992 index = sjdl2.Index("InputDataList");
1993 if (index>=0) {
1994 index1 = sjdl2.Index("\n", index);
1995 sjdl2.Remove(index, index1-index+1);
1996 }
f47d5cb4 1997 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1998 // Write jdl to file
a8739e8a 1999 ofstream out;
d2a409b2 2000 out.open(fJDLName.Data(), ios::out);
a8739e8a 2001 if (out.bad()) {
d3b18c4c 2002 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2003 return kFALSE;
2004 }
2005 out << sjdl << endl;
b3e07543 2006 out.close();
0f389141 2007 TString mergeJDLName = fExecutable;
2008 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2009 if (fMergeViaJDL) {
2010 ofstream out1;
2011 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2012 if (out1.bad()) {
d3b18c4c 2013 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2014 return kFALSE;
2015 }
2016 out1 << sjdl1 << endl;
b3e07543 2017 out1.close();
2018 ofstream out2;
2019 TString finalJDL = mergeJDLName;
2020 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2021 out2.open(finalJDL.Data(), ios::out);
2022 if (out2.bad()) {
2023 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2024 return kFALSE;
2025 }
2026 out2 << sjdl2 << endl;
2027 out2.close();
0f389141 2028 }
a8739e8a 2029
2030 // Copy jdl to grid workspace
2031 if (!copy) {
d3b18c4c 2032 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2033 } else {
d2a409b2 2034 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2035 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2036 TString finalJDL = mergeJDLName;
2037 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2038 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2039 if (fProductionMode) {
b5fe9cba 2040 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2041 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2042 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2043 }
d2a409b2 2044 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2045 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2046 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2047 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2048// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2049 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2050 Fatal("","Terminating");
0f389141 2051 if (fMergeViaJDL) {
b3e07543 2052 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2053// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2054// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2055 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2056 Fatal("","Terminating");
2057 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2058 Fatal("","Terminating");
0f389141 2059 }
a8739e8a 2060 }
2061 return kTRUE;
2062}
2063
c57f56b7 2064//______________________________________________________________________________
5513444a 2065Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2066{
2067// Returns true if file exists.
5513444a 2068 if (!gGrid) return kFALSE;
2ea55496 2069 TString slfn = lfn;
2070 slfn.ReplaceAll("alien://","");
2071 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2072 if (!res) return kFALSE;
2073 TMap *map = dynamic_cast<TMap*>(res->At(0));
2074 if (!map) {
2075 delete res;
2076 return kFALSE;
2077 }
2078 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2079 if (!objs || !objs->GetString().Length()) {
2080 delete res;
2081 return kFALSE;
2082 }
2083 delete res;
2084 return kTRUE;
2085}
2086
923e2ca5 2087//______________________________________________________________________________
2088Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2089{
2090// Returns true if directory exists. Can be also a path.
2091 if (!gGrid) return kFALSE;
2092 // Check if dirname is a path
2093 TString dirstripped = dirname;
2094 dirstripped = dirstripped.Strip();
2095 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2096 TString dir = gSystem->BaseName(dirstripped);
2097 dir += "/";
2098 TString path = gSystem->DirName(dirstripped);
2099 TGridResult *res = gGrid->Ls(path, "-F");
2100 if (!res) return kFALSE;
2101 TIter next(res);
2102 TMap *map;
2103 TObject *obj;
2104 while ((map=dynamic_cast<TMap*>(next()))) {
2105 obj = map->GetValue("name");
2106 if (!obj) break;
2107 if (dir == obj->GetName()) {
2108 delete res;
2109 return kTRUE;
2110 }
2111 }
2112 delete res;
2113 return kFALSE;
2114}
2115
c57f56b7 2116//______________________________________________________________________________
a2f5fc01 2117void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2118{
2119// Check input data type.
a2f5fc01 2120 isCollection = kFALSE;
2121 isXml = kFALSE;
2122 useTags = kFALSE;
c57f56b7 2123 if (!gGrid) {
2124 Error("CheckDataType", "No connection to grid");
2125 return;
2126 }
a2f5fc01 2127 isCollection = IsCollection(lfn);
c57f56b7 2128 TString msg = "\n##### file: ";
2129 msg += lfn;
a2f5fc01 2130 if (isCollection) {
c57f56b7 2131 msg += " type: raw_collection;";
2132 // special treatment for collections
a2f5fc01 2133 isXml = kFALSE;
c57f56b7 2134 // check for tag files in the collection
2135 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2136 if (!res) {
2137 msg += " using_tags: No (unknown)";
23329835 2138 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2139 return;
2140 }
2141 const char* typeStr = res->GetKey(0, "origLFN");
2142 if (!typeStr || !strlen(typeStr)) {
2143 msg += " using_tags: No (unknown)";
23329835 2144 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2145 return;
2146 }
2147 TString file = typeStr;
a2f5fc01 2148 useTags = file.Contains(".tag");
2149 if (useTags) msg += " using_tags: Yes";
c57f56b7 2150 else msg += " using_tags: No";
23329835 2151 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2152 return;
2153 }
2154 TString slfn(lfn);
2155 slfn.ToLower();
a2f5fc01 2156 isXml = slfn.Contains(".xml");
2157 if (isXml) {
c57f56b7 2158 // Open xml collection and check if there are tag files inside
2159 msg += " type: xml_collection;";
2160 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2161 if (!coll) {
2162 msg += " using_tags: No (unknown)";
23329835 2163 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2164 return;
2165 }
2166 TMap *map = coll->Next();
2167 if (!map) {
2168 msg += " using_tags: No (unknown)";
23329835 2169 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2170 return;
2171 }
2172 map = (TMap*)map->GetValue("");
2173 TString file;
2174 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2175 useTags = file.Contains(".tag");
c57f56b7 2176 delete coll;
a2f5fc01 2177 if (useTags) msg += " using_tags: Yes";
c57f56b7 2178 else msg += " using_tags: No";
23329835 2179 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2180 return;
2181 }
a2f5fc01 2182 useTags = slfn.Contains(".tag");
c57f56b7 2183 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2184 else msg += " type: unknown file;";
a2f5fc01 2185 if (useTags) msg += " using_tags: Yes";
c57f56b7 2186 else msg += " using_tags: No";
23329835 2187 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2188}
2189
4e5c5506 2190//______________________________________________________________________________
2191void AliAnalysisAlien::EnablePackage(const char *package)
2192{
2193// Enables a par file supposed to exist in the current directory.
2194 TString pkg(package);
2195 pkg.ReplaceAll(".par", "");
2196 pkg += ".par";
2197 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2198 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2199 return;
2200 }
2201 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2202 Info("EnablePackage", "AliEn plugin will use .par packages");
2203 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2204 if (!fPackages) {
2205 fPackages = new TObjArray();
2206 fPackages->SetOwner();
2207 }
2208 fPackages->Add(new TObjString(pkg));
2209}
2210
3bdcb562 2211//______________________________________________________________________________
2212TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2213{
2214// Make a tree from files having the location specified in fFileForTestMode.
2215// Inspired from JF's CreateESDChain.
2216 if (fFileForTestMode.IsNull()) {
2217 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2218 return NULL;
2219 }
2220 if (gSystem->AccessPathName(fFileForTestMode)) {
2221 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2222 return NULL;
2223 }
2224 // Open the file
2225 ifstream in;
2226 in.open(fFileForTestMode);
2227 Int_t count = 0;
2228 // Read the input list of files and add them to the chain
b505f1f8 2229 TString line;
2230 TChain *chain = new TChain(treeName);
2231 TChain *chainFriend = 0;
2232 if (!fFriendChainName.IsNull()) chainFriend = new TChain(treeName);
2233 while (in.good())
2234 {
3bdcb562 2235 in >> line;
2236 if (line.IsNull()) continue;
2237 if (count++ == fNtestFiles) break;
2238 TString esdFile(line);
2239 TFile *file = TFile::Open(esdFile);
b505f1f8 2240 if (file && !file->IsZombie()) {
2241 chain->Add(esdFile);
3bdcb562 2242 file->Close();
b505f1f8 2243 if (!fFriendChainName.IsNull()) {
2244 esdFile.ReplaceAll("AliAOD.root", fFriendChainName.Data());
2245 esdFile.ReplaceAll("AliAODs.root", fFriendChainName.Data());
2246 file = TFile::Open(esdFile);
2247 if (file && !file->IsZombie()) {
2248 file->Close();
2249 chainFriend->Add(esdFile);
2250 } else {
2251 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2252 return 0;
2253 }
2254 }
3bdcb562 2255 } else {
2256 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2257 }
b505f1f8 2258 }
2259 in.close();
2260 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2261 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2262 delete chain;
b505f1f8 2263 delete chainFriend;
3bdcb562 2264 return NULL;
b505f1f8 2265 }
3bdcb562 2266// chain->ls();
b505f1f8 2267 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2268 return chain;
3bdcb562 2269}
2270
d2a409b2 2271//______________________________________________________________________________
2272const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2273{
2274// Get job status for all jobs with jobid>jobidstart.
2275 static char mstatus[20];
2276 mstatus[0] = '\0';
2277 nrunning = 0;
2278 nwaiting = 0;
2279 nerror = 0;
2280 ndone = 0;
2281 TGridJobStatusList *list = gGrid->Ps("");
2282 if (!list) return mstatus;
2283 Int_t nentries = list->GetSize();
2284 TGridJobStatus *status;
2285 Int_t pid;
2286 for (Int_t ijob=0; ijob<nentries; ijob++) {
2287 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2288 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2289 if (pid<jobidstart) continue;
2290 if (pid == lastid) {
8ddad121 2291 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2292 }
2293 switch (status->GetStatus()) {
2294 case TGridJobStatus::kWAITING:
2295 nwaiting++; break;
2296 case TGridJobStatus::kRUNNING:
2297 nrunning++; break;
2298 case TGridJobStatus::kABORTED:
2299 case TGridJobStatus::kFAIL:
2300 case TGridJobStatus::kUNKNOWN:
2301 nerror++; break;
2302 case TGridJobStatus::kDONE:
2303 ndone++;
2304 }
2305 }
2306 list->Delete();
2307 delete list;
2308 return mstatus;
2309}
2310
c57f56b7 2311//______________________________________________________________________________
2312Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2313{
2314// Returns true if file is a collection. Functionality duplicated from
2315// TAlien::Type() because we don't want to directly depend on TAlien.
2316 if (!gGrid) {
2317 Error("IsCollection", "No connection to grid");
2318 return kFALSE;
2319 }
2320 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2321 if (!res) return kFALSE;
2322 const char* typeStr = res->GetKey(0, "type");
2323 if (!typeStr || !strlen(typeStr)) return kFALSE;
2324 if (!strcmp(typeStr, "collection")) return kTRUE;
2325 delete res;
2326 return kFALSE;
2327}
2328
fe2d7fc2 2329//______________________________________________________________________________
2330Bool_t AliAnalysisAlien::IsSingleOutput() const
2331{
2332// Check if single-ouput option is on.
2333 return (!fOutputSingle.IsNull());
2334}
2335
16a4353c 2336//______________________________________________________________________________
2337void AliAnalysisAlien::Print(Option_t *) const
2338{
2339// Print current plugin settings.
84fcd93f 2340 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2341 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2342 if (mgr && mgr->IsProofMode()) {
2343 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2344 if (TestBit(AliAnalysisGrid::kTest))
2345 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2346 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2347 if (!fProofDataSet.IsNull())
2348 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2349 if (fProofReset==1)
2350 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2351 if (fProofReset>1)
2352 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2353 if (!fRootVersionForProof.IsNull())
2354 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2355 else
2356 printf("= ROOT version requested________________________ default\n");
2357 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2358 if (!fAliRootMode.IsNull())
2359 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2360 if (fNproofWorkers)
2361 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2362 if (fNproofWorkersPerSlave)
2363 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2364 if (TestSpecialBit(kClearPackages))
2365 printf("= ClearPackages requested...\n");
2366 if (fIncludePath.Data())
2367 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2368 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2369 if (fPackages && fPackages->GetEntries()) {
2370 TIter next(fPackages);
2371 TObject *obj;
2372 TString list;
2373 while ((obj=next())) list += obj->GetName();
2374 printf("= Par files to be used: ________________________ %s\n", list.Data());
2375 }
2376 if (TestSpecialBit(kProofConnectGrid))
2377 printf("= Requested PROOF connection to grid\n");
2378 return;
2379 }
e1c22e21 2380 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2381 if (fOverwriteMode) {
2382 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2383 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2384 }
348be253 2385 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2386 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2387 printf("= Production mode:______________________________ %d\n", fProductionMode);
2388 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2389 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2390 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2391 if (fUser.Length())
84fcd93f 2392 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2393 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2394 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2395 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2396 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2397 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2398 if (fRunNumbers.Length())
84fcd93f 2399 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2400 if (fRunRange[0])
74013535 2401 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2402 if (!fRunRange[0] && !fRunNumbers.Length()) {
2403 TIter next(fInputFiles);
2404 TObject *obj;
2405 TString list;
2406 while ((obj=next())) list += obj->GetName();
84fcd93f 2407 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2408 }
2409 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2410 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2411 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2412 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2413 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2414 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2415 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2416 printf("=====================================================================\n");
2417 printf("= Job price: ___________________________________ %d\n", fPrice);
2418 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2419 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2420 if (fMaxInitFailed>0)
84fcd93f 2421 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2422 if (fMasterResubmitThreshold>0)
84fcd93f 2423 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2424 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2425 if (fNrunsPerMaster>0)
84fcd93f 2426 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2427 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2428 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2429 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2430 if (fArguments.Length())
84fcd93f 2431 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2432 if (fExecutableArgs.Length())
2433 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2434 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2435 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2436 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2437 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2438 if (fDatasetName)
84fcd93f 2439 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2440 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2441 if (fIncludePath.Data())
84fcd93f 2442 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2443 if (fCloseSE.Length())
84fcd93f 2444 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2445 if (fFriendChainName.Length())
84fcd93f 2446 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2447 if (fPackages && fPackages->GetEntries()) {
16a4353c 2448 TIter next(fPackages);
2449 TObject *obj;
2450 TString list;
2451 while ((obj=next())) list += obj->GetName();
84fcd93f 2452 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2453 }
2454}
2455
c57f56b7 2456//______________________________________________________________________________
2457void AliAnalysisAlien::SetDefaults()
2458{
2459// Set default values for everything. What cannot be filled will be left empty.
2460 if (fGridJDL) delete fGridJDL;
2461 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2462 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2463 fPrice = 1;
2464 fTTL = 30000;
2465 fSplitMaxInputFileNumber = 100;
2466 fMaxInitFailed = 0;
2467 fMasterResubmitThreshold = 0;
bb885a9e 2468 fNtestFiles = 10;
149d288c 2469 fNreplicas = 2;
a8739e8a 2470 fRunRange[0] = 0;
2471 fRunRange[1] = 0;
8e3bd06a 2472 fRunPrefix = "%d";
319593fb 2473 fNrunsPerMaster = 1;
16a4353c 2474 fMaxMergeFiles = 100;
c57f56b7 2475 fRunNumbers = "";
2476 fExecutable = "analysis.sh";
0a1c1f7f 2477 fExecutableCommand = "root -b -q";
c57f56b7 2478 fArguments = "";
631c0b05 2479 fExecutableArgs = "";
c57f56b7 2480 fAnalysisMacro = "myAnalysis.C";
2481 fAnalysisSource = "";
2482 fAdditionalLibs = "";
2483 fSplitMode = "se";
2484 fAPIVersion = "";
2485 fROOTVersion = "";
2486 fAliROOTVersion = "";
2487 fUser = ""; // Your alien user name
2488 fGridWorkingDir = "";
2489 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2490 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2491 fFriendChainName = "";
c57f56b7 2492 fGridOutputDir = "output";
b83f84f0 2493 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2494 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2495 fInputFormat = "xml-single";
2496 fJDLName = "analysis.jdl";
c6cb3634 2497 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2498 fMergeExcludes = "";
0f389141 2499 fMergeViaJDL = 0;
348be253 2500 SetUseCopy(kTRUE);
2501 SetCheckCopy(kTRUE);
149d288c 2502 SetDefaultOutputs(kTRUE);
e1c22e21 2503 fOverwriteMode = 1;
c57f56b7 2504}
2505
0f389141 2506//______________________________________________________________________________
b3e07543 2507Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2508{
b3e07543 2509// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2510 // First check if the result is already in the output directory.
2511 if (FileExists(Form("%s/%s",aliendir,filename))) {
2512 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2513 return kFALSE;
2514 }
b3e07543 2515 // Now check the last stage done.
2516 Int_t stage = 0;
2517 while (1) {
2518 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2519 stage++;
2520 }
2521 // Next stage of merging
2522 stage++;
2523 TString pattern = "*root_archive.zip";
2524 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2525 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2526 if (res) delete res;
2527 // Write standard output to file
2528 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2529 // Count the number of files inside
2530 ifstream ifile;
2531 ifile.open(Form("Stage_%d.xml",stage));
2532 if (!ifile.good()) {
2533 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2534 return kFALSE;
b3e07543 2535 }
2536 TString line;
2537 Int_t nfiles = 0;
2538 while (!ifile.eof()) {
2539 ifile >> line;
2540 if (line.Contains("/event")) nfiles++;
7c2cd90a 2541 }
b3e07543 2542 ifile.close();
2543 if (!nfiles) {
2544 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2545 return kFALSE;
2546 } else {
2547 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2548 }
b3e07543 2549 // Copy the file in the output directory
2550 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2551// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2552 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2553 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2554 // Check if this is the last stage to be done.
2555 Bool_t laststage = (nfiles<nperchunk);
2556 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2557 if (laststage) {
2558 printf("### Submiting final merging stage %d\n", stage);
2559 TString finalJDL = jdl;
2560 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2561 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2562 Int_t jobId = SubmitSingleJob(query);
2563 if (!jobId) return kFALSE;
2564 } else {
2565 printf("### Submiting merging stage %d\n", stage);
03335324 2566 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2567 Int_t jobId = SubmitSingleJob(query);
2568 if (!jobId) return kFALSE;
7c2cd90a 2569 }
b3e07543 2570 return kTRUE;
2571}
7c2cd90a 2572
105631b5 2573//______________________________________________________________________________
2574AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2575{
2576// Loat the analysis manager from a file.
2577 TFile *file = TFile::Open(fname);
2578 if (!file) {
2579 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2580 return 0;
2581 }
2582 TIter nextkey(file->GetListOfKeys());
2583 AliAnalysisManager *mgr = 0;
2584 TKey *key;
2585 while ((key=(TKey*)nextkey())) {
2586 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2587 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2588 }
2589 if (!mgr)
2590 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2591 return mgr;
2592}
2593
7c2cd90a 2594//______________________________________________________________________________
2595Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2596{
2597// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2598 if (!gGrid) return 0;
2599 printf("=> %s ------> ",query);
2600 TGridResult *res = gGrid->Command(query);
2601 if (!res) return 0;
2602 TString jobId = res->GetKey(0,"jobId");
2603 delete res;
2604 if (jobId.IsNull()) {
2605 printf("submission failed. Reason:\n");
2606 gGrid->Stdout();
2607 gGrid->Stderr();
2608 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2609 return 0;
2610 }
2611 printf(" Job id: %s\n", jobId.Data());
2612 return atoi(jobId);
2613}
2614
2615//______________________________________________________________________________
b3e07543 2616Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2617{
b3e07543 2618// Merge given output files from basedir. Basedir can be an alien output directory
2619// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2620// files in a group (ignored for xml input). Merging can be done in stages:
2621// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2622// stage=1 : works with an xml of all root_archive.zip in the output directory
2623// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2624 TString outputFile = output;
0f389141 2625 TString command;
a2f5fc01 2626 TString outputChunk;
2627 TString previousChunk = "";
b3e07543 2628 TObjArray *listoffiles = new TObjArray();
2629// listoffiles->SetOwner();
a2f5fc01 2630 Int_t countChunk = 0;
2631 Int_t countZero = nmaxmerge;
0f389141 2632 Bool_t merged = kTRUE;
a2f5fc01 2633 Int_t index = outputFile.Index("@");
2634 if (index > 0) outputFile.Remove(index);
7c2cd90a 2635 TString inputFile = outputFile;
b3e07543 2636 TString sbasedir = basedir;
2637 if (sbasedir.Contains(".xml")) {
2638 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2639 nmaxmerge = 9999999;
2640 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2641 if (!coll) {
2642 ::Error("MergeOutput", "Input XML collection empty.");
2643 return kFALSE;
2644 }
2645 // Iterate grid collection
2646 while (coll->Next()) {
2647 TString fname = gSystem->DirName(coll->GetTURL());
2648 fname += "/";
2649 fname += inputFile;
2650 listoffiles->Add(new TNamed(fname.Data(),""));
2651 }
2652 } else {
2653 command = Form("find %s/ *%s", basedir, inputFile.Data());
2654 printf("command: %s\n", command.Data());
2655 TGridResult *res = gGrid->Command(command);
2656 if (!res) {
2657 ::Error("MergeOutput","No result for the find command\n");
2658 delete listoffiles;
2659 return kFALSE;
2660 }
2661 TIter nextmap(res);
2662 TMap *map = 0;
2663 while ((map=(TMap*)nextmap())) {
2664 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2665 if (!objs || !objs->GetString().Length()) {
2666 // Nothing found - skip this output
2667 delete res;
2668 delete listoffiles;
2669 return kFALSE;
2670 }
2671 listoffiles->Add(new TNamed(objs->GetName(),""));
2672 }
2673 delete res;
2674 }
2675 if (!listoffiles->GetEntries()) {
7c2cd90a 2676 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2677 delete listoffiles;
0f389141 2678 return kFALSE;
2679 }
2680
2681 TFileMerger *fm = 0;
b3e07543 2682 TIter next0(listoffiles);
2683 TObjArray *listoffilestmp = new TObjArray();
2684 listoffilestmp->SetOwner();
2685 TObject *nextfile;
2686 TString snextfile;
2687 // Keep only the files at upper level
2688 Int_t countChar = 0;
2689 while ((nextfile=next0())) {
2690 snextfile = nextfile->GetName();
2691 Int_t crtCount = snextfile.CountChar('/');
2692 if (nextfile == listoffiles->First()) countChar = crtCount;
2693 if (crtCount < countChar) countChar = crtCount;
2694 }
2695 next0.Reset();
2696 while ((nextfile=next0())) {
2697 snextfile = nextfile->GetName();
2698 Int_t crtCount = snextfile.CountChar('/');
2699 if (crtCount > countChar) {
2700 delete nextfile;
2701 continue;
2702 }
2703 listoffilestmp->Add(nextfile);
2704 }
2705 delete listoffiles;
2706 listoffiles = listoffilestmp; // Now contains 'good' files
2707 listoffiles->Print();
2708 TIter next(listoffiles);
7c2cd90a 2709 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2710 outputChunk = outputFile;
2711 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2712 // Check for existent temporary merge files
e1c22e21 2713 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2714 // Preserve old merging functionality for stage 0.
2715 if (stage==0) {
2716 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2717 while (1) {
2718 // Skip as many input files as in a chunk
2ea55496 2719 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2720 nextfile = next();
2721 if (!nextfile) {
2ea55496 2722 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2723 delete listoffiles;
2ea55496 2724 return kFALSE;
2725 }
b3e07543 2726 snextfile = nextfile->GetName();
7c2cd90a 2727 }
2728 outputChunk = outputFile;
2729 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2730 countChunk++;
2731 if (gSystem->AccessPathName(outputChunk)) continue;
2732 // Merged file with chunks up to <countChunk> found
2733 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2734 previousChunk = outputChunk;
2735 break;
2736 }
2737 }
2738 countZero = nmaxmerge;
2739
b3e07543 2740 while ((nextfile=next())) {
2741 snextfile = nextfile->GetName();
2ea55496 2742 // Loop 'find' results and get next LFN
7c2cd90a 2743 if (countZero == nmaxmerge) {
2744 // First file in chunk - create file merger and add previous chunk if any.
2b895f34 2745 fm = new TFileMerger(kTRUE);
7c2cd90a 2746 fm->SetFastMethod(kTRUE);
2747 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2748 outputChunk = outputFile;
2749 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2750 }
2751 // If last file found, put merged results in the output file
b3e07543 2752 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2753 // Add file to be merged and decrement chunk counter.
b3e07543 2754 fm->AddFile(snextfile);
7c2cd90a 2755 countZero--;
b3e07543 2756 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2757 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2758 // Nothing found - skip this output
2759 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2760 merged = kFALSE;
2761 break;
7c2cd90a 2762 }
2763 fm->OutputFile(outputChunk);
2764 // Merge the outputs, then go to next chunk
2765 if (!fm->Merge()) {
2766 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2767 merged = kFALSE;
2768 break;
7c2cd90a 2769 } else {
2770 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2771 gSystem->Unlink(previousChunk);
2772 }
b3e07543 2773 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2774 countChunk++;
2775 countZero = nmaxmerge;
2776 previousChunk = outputChunk;
0f389141 2777 }
0f389141 2778 }
b3e07543 2779 delete listoffiles;
2780 delete fm;
7c2cd90a 2781 return merged;
2782 }
2783 // Merging stage different than 0.
2784 // Move to the begining of the requested chunk.
2b895f34 2785 fm = new TFileMerger(kTRUE);
7c2cd90a 2786 fm->SetFastMethod(kTRUE);
b3e07543 2787 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2788 delete listoffiles;
7c2cd90a 2789 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2790 // Nothing found - skip this output
2791 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2792 delete fm;
2793 return kFALSE;
2794 }
b3e07543 2795 fm->OutputFile(outputFile);
7c2cd90a 2796 // Merge the outputs
2797 if (!fm->Merge()) {
2798 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2799 delete fm;
2800 return kFALSE;
2801 } else {
b3e07543 2802 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2803 }
7c2cd90a 2804 delete fm;
2805 return kTRUE;
0f389141 2806}
2807
c57f56b7 2808//______________________________________________________________________________
2809Bool_t AliAnalysisAlien::MergeOutputs()
2810{
2811// Merge analysis outputs existing in the AliEn space.
2812 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2813 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2814 if (!Connect()) {
2815 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2816 return kFALSE;
0f389141 2817 }
c9e8f7fd 2818 if (fMergeViaJDL) {
2819 if (!TestBit(AliAnalysisGrid::kMerge)) {
2820 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2821 return kFALSE;
2822 }
2823 if (fProductionMode) {
2824 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2825 return kFALSE;
2826 }
0f389141 2827 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2828 if (!SubmitMerging()) return kFALSE;
0f389141 2829 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2830 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2831 return kFALSE;
c57f56b7 2832 }
2833 // Get the output path
dc030a24 2834 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2835 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2836 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2837 return kFALSE;
2838 }
2839 if (!fOutputFiles.Length()) {
2840 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2841 return kFALSE;
0f389141 2842 }
2843 // Check if fast read option was requested
e1c22e21 2844 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2845 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2846 if (fFastReadOption) {
d3339be3 2847 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2848 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2849 gEnv->SetValue("XNet.ConnectTimeout",50);
2850 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2851 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2852 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2853 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2854 }
e8b839ab 2855 // Make sure we change the temporary directory
2856 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2857 // Set temporary compilation directory to current one
2858 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2859 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2860 TIter next(list);
2861 TObjString *str;
a2f5fc01 2862 TString outputFile;
c57f56b7 2863 Bool_t merged = kTRUE;
2864 while((str=(TObjString*)next())) {
a2f5fc01 2865 outputFile = str->GetString();
2866 Int_t index = outputFile.Index("@");
2867 if (index > 0) outputFile.Remove(index);
2868 TString outputChunk = outputFile;
2869 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2870 // Skip already merged outputs
a2f5fc01 2871 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2872 if (fOverwriteMode) {
a2f5fc01 2873 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2874 gSystem->Unlink(outputFile);
2875 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2876 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2877 outputChunk.Data());
2878 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2879 }
2880 } else {
a2f5fc01 2881 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2882 continue;
2883 }
2884 } else {
a2f5fc01 2885 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2886 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2887 outputChunk.Data());
2888 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2889 }
2890 }
37d21c01 2891 if (fMergeExcludes.Contains(outputFile.Data()) ||
2892 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 2893 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2894 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2895 if (!merged) {
2896 Error("MergeOutputs", "Terminate() will NOT be executed");
2897 return kFALSE;
ff07ec61 2898 }
2899 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2900 if (fileOpened) fileOpened->Close();
c57f56b7 2901 }
0f389141 2902 return kTRUE;
c57f56b7 2903}
2904
bb885a9e 2905//______________________________________________________________________________
2906void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2907{
2908// Use the output files connected to output containers from the analysis manager
2909// rather than the files defined by SetOutputFiles
2910 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2911 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2912 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2913}
2914
149d288c 2915//______________________________________________________________________________
2916void AliAnalysisAlien::SetOutputFiles(const char *list)
2917{
2918// Manually set the output files list.
2919// Removes duplicates. Not allowed if default outputs are not disabled.
2920 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2921 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2922 return;
2923 }
2924 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2925 fOutputFiles = "";
2926 TString slist = list;
2927 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2928 TObjArray *arr = slist.Tokenize(" ");
2929 TObjString *os;
2930 TIter next(arr);
2931 TString sout;
2932 while ((os=(TObjString*)next())) {
2933 sout = os->GetString();
2934 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2935 if (fOutputFiles.Contains(sout)) continue;
2936 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2937 fOutputFiles += sout;
2938 }
2939 delete arr;
f790bc1b 2940}
149d288c 2941
2942//______________________________________________________________________________
2943void AliAnalysisAlien::SetOutputArchive(const char *list)
2944{
2945// Manually set the output archive list. Free text - you are on your own...
2946// Not allowed if default outputs are not disabled.
2947 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2948 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2949 return;
2950 }
2951 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2952 fOutputArchive = list;
2953}
2954
2955//______________________________________________________________________________
2956void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2957{
2958// Setting a prefered output SE is not allowed anymore.
2959 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2960}
2961
28ce6c9c 2962//______________________________________________________________________________
2963void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2964{
2965// Set some PROOF special parameter.
2966 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2967 if (pair) {
2968 TObject *old = pair->Key();
2969 TObject *val = pair->Value();
2970 fProofParam.Remove(old);
2971 delete old;
2972 delete val;
2973 }
2974 fProofParam.Add(new TObjString(pname), new TObjString(value));
2975}
2976
2977//______________________________________________________________________________
2978const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2979{
2980// Returns a special PROOF parameter.
2981 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2982 if (!pair) return 0;
2983 return pair->Value()->GetName();
2984}
2985
5513444a 2986//______________________________________________________________________________
2987Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2988{
2989// Start remote grid analysis.
3bdcb562 2990 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2991 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2992 if (!mgr || !mgr->IsInitialized()) {
2993 Error("StartAnalysis", "You need an initialized analysis manager for this");
2994 return kFALSE;
2995 }
2996 // Are we in PROOF mode ?
2997 if (mgr->IsProofMode()) {
baa34407 2998 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2999 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3000 if (fProofCluster.IsNull()) {
3001 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3002 return kFALSE;
3003 }
3004 if (fProofDataSet.IsNull() && !testMode) {
3005 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3006 return kFALSE;
3007 }
3008 // Set the needed environment
3009 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3010 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3011 if (fProofReset && !testMode) {
3012 if (fProofReset==1) {
3013 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3014 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3015 } else {
3016 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3017 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3018 }
3019 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3020 return kFALSE;
3021 }
baa34407 3022
3023 if (!testMode) {
3024 // Check if there is an old active session
3025 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3026 if (nsessions) {
3027 Error("StartAnalysis","You have to reset your old session first\n");
3028 return kFALSE;
3029 }
53911ca6 3030 }
3bdcb562 3031 // Do we need to change the ROOT version ? The success of this cannot be checked.
3032 if (!fRootVersionForProof.IsNull() && !testMode) {
3033 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
3034 fProofCluster.Data(), fRootVersionForProof.Data()));
3035 }
3036 // Connect to PROOF and check the status
3037 Long_t proof = 0;
27734f0e 3038 TString sworkers;
3039 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3040 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3041 if (!testMode) {
27734f0e 3042 if (!sworkers.IsNull())
3043 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3044 else
3045 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3046 } else {
3047 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3048 if (!proof) {
3049 Error("StartAnalysis", "Could not start PROOF in test mode");
3050 return kFALSE;
3051 }
3052 }
3053 if (!proof) {
3054 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3055 return kFALSE;
3056 }
27734f0e 3057 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3058 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3059 // Set proof special parameters if any
3060 TIter nextpp(&fProofParam);
3061 TObject *proofparam;
3062 while ((proofparam=nextpp())) {
3063 TString svalue = GetProofParameter(proofparam->GetName());
3064 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3065 }
3bdcb562 3066 // Is dataset existing ?
3067 if (!testMode) {
3068 TString dataset = fProofDataSet;
3069 Int_t index = dataset.Index("#");
3070 if (index>=0) dataset.Remove(index);
830acc4c 3071// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3072// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3073// return kFALSE;
3074// }
3075// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3076 }
3077 // Is ClearPackages() needed ?
3078 if (TestSpecialBit(kClearPackages)) {
3079 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3080 gROOT->ProcessLine("gProof->ClearPackages();");
3081 }
3082 // Is a given aliroot mode requested ?
3083 TList optionsList;
1ac846e9 3084 TString parLibs;
830acc4c 3085 if (!fAliRootMode.IsNull()) {
3bdcb562 3086 TString alirootMode = fAliRootMode;
3087 if (alirootMode == "default") alirootMode = "";
3088 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3089 optionsList.SetOwner();
3090 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3091 // Check the additional libs to be loaded
3092 TString extraLibs;
1ac846e9 3093 Bool_t parMode = kFALSE;
dc1ffbcb 3094 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3095 // Parse the extra libs for .so
3096 if (fAdditionalLibs.Length()) {
3097 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3098 TIter next(list);
3099 TObjString *str;
1ac846e9 3100 while((str=(TObjString*)next())) {
3101 if (str->GetString().Contains(".so")) {
3102 if (parMode) {
3103 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3104 break;
3105 }
3106 TString stmp = str->GetName();
3107 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3108 stmp.ReplaceAll(".so","");
3109 if (!extraLibs.IsNull()) extraLibs += ":";
3110 extraLibs += stmp;
3111 continue;
3112 }
3113 if (str->GetString().Contains(".par")) {
3114 // The first par file found in the list will not allow any further .so
3115 parMode = kTRUE;
3116 if (!parLibs.IsNull()) parLibs += ":";
3117 parLibs += str->GetName();
3118 continue;
3119 }
3bdcb562 3120 }
3121 if (list) delete list;
3122 }
dc030a24 3123 if (!extraLibs.IsNull()) {
3124 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3125 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3126 }
830acc4c 3127 // Check extra includes
3128 if (!fIncludePath.IsNull()) {
3129 TString includePath = fIncludePath;
3130 includePath.ReplaceAll(" ",":");
dc030a24 3131 includePath.ReplaceAll("$ALICE_ROOT/","");
3132 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3133 includePath.ReplaceAll("-I","");
dc030a24 3134 includePath.Remove(TString::kTrailing, ':');
830acc4c 3135 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3136 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3137 }
27734f0e 3138 // Check if connection to grid is requested
3139 if (TestSpecialBit(kProofConnectGrid))
3140 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3141 // Enable AliRoot par
3142 if (testMode) {
3143 // Enable proof lite package
3144 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3145 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3146 TNamed *obj = (TNamed*)optionsList.At(i);
3147 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3148 }
3149 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3150 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3151 Info("StartAnalysis", "AliRootProofLite enabled");
3152 } else {
3153 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3154 return kFALSE;
3155 }
3156 } else {
dc030a24 3157 if ( ! fAliROOTVersion.IsNull() ) {
3158 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3159 fAliROOTVersion.Data(), &optionsList))) {
3160 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3161 return kFALSE;
3162 }
3163 }
27734f0e 3164 }
1ac846e9 3165 // Enable first par files from fAdditionalLibs
3166 if (!parLibs.IsNull()) {
3167 TObjArray *list = parLibs.Tokenize(":");
3168 TIter next(list);
3169 TObjString *package;
3170 while((package=(TObjString*)next())) {
40501bb7 3171 TString spkg = package->GetName();
3172 spkg.ReplaceAll(".par", "");
3173 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3174 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3175 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3176 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3177 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3178 return kFALSE;
3179 }
3180 } else {
3181 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3182 return kFALSE;
3183 }
3184 }
3185 if (list) delete list;
3186 }
3bdcb562 3187 } else {
3188 if (fAdditionalLibs.Contains(".so") && !testMode) {
3189 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3190 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3191 return kFALSE;
3192 }
3193 }
3194 // Enable par files if requested
3195 if (fPackages && fPackages->GetEntries()) {
3196 TIter next(fPackages);
3197 TObject *package;
3198 while ((package=next())) {
1ac846e9 3199 // Skip packages already enabled
3200 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3201 TString spkg = package->GetName();
3202 spkg.ReplaceAll(".par", "");
3203 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3204 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3205 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3206 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3207 return kFALSE;
3208 }
3209 } else {
3210 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3211 return kFALSE;
3212 }
3213 }
3214 }
3215 // Do we need to load analysis source files ?
3216 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3217 if (fAnalysisSource.Length()) {
3218 TObjArray *list = fAnalysisSource.Tokenize(" ");
3219 TIter next(list);
3220 TObjString *str;
3221 while((str=(TObjString*)next())) {
3222 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3223 }
3bdcb562 3224 if (list) delete list;
3225 }
830acc4c 3226 if (testMode) {
3227 // Register dataset to proof lite.
3228 if (fFileForTestMode.IsNull()) {
3229 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3230 return kFALSE;
3231 }
3232 if (gSystem->AccessPathName(fFileForTestMode)) {
3233 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3234 return kFALSE;
3235 }
3236 TFileCollection *coll = new TFileCollection();
3237 coll->AddFromFile(fFileForTestMode);
8ddad121 3238 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3239 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3240 }
3bdcb562 3241 return kTRUE;
3242 }
c57f56b7 3243
43da816a 3244 // Check if output files have to be taken from the analysis manager
3245 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3246 // Add output files and AOD files
3247 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3248 // Add extra files registered to the analysis manager
f790bc1b 3249 TString extra = GetListOfFiles("ext");
3250 if (!extra.IsNull()) {
149d288c 3251 extra.ReplaceAll(".root", "*.root");
f790bc1b 3252 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3253 fOutputFiles += extra;
84fcd93f 3254 }
149d288c 3255 // Compose the output archive.
b83f84f0 3256 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3257 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3258 }
f7b1cbc2 3259// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3260 if (TestBit(AliAnalysisGrid::kOffline)) {
3261 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3262 \n there nor any job run. You can revise the JDL and analysis \
3263 \n macro then run the same in \"submit\" mode.");
3264 } else if (TestBit(AliAnalysisGrid::kTest)) {
3265 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3266 \n dataset.");
3267 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3268 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3269 \n space and job submitted.");
3270 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3271 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3272 if (fMergeViaJDL) CheckInputData();
5513444a 3273 return kTRUE;
c57f56b7 3274 } else {
3275 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3276 }
3277
348be253 3278 Print();
c57f56b7 3279 if (!Connect()) {
3280 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3281 return kFALSE;
16a4353c 3282 }
d3b18c4c 3283 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3284 if (!CheckInputData()) {
3285 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3286 return kFALSE;
c57f56b7 3287 }
d3339be3 3288 if (!CreateDataset(fDataPattern)) {
3289 TString serror;
3290 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3291 if (fRunNumbers.Length()) serror = "run numbers";
3292 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3293 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3294 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3295 return kFALSE;
3296 }
21ca8e59 3297 WriteAnalysisFile();
c57f56b7 3298 WriteAnalysisMacro();
3299 WriteExecutable();
3300 WriteValidationScript();
0f389141 3301 if (fMergeViaJDL) {
3302 WriteMergingMacro();
3303 WriteMergeExecutable();
3304 WriteValidationScript(kTRUE);
3305 }
5513444a 3306 if (!CreateJDL()) return kFALSE;
3307 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3308 if (testMode) {
c57f56b7 3309 // Locally testing the analysis
3310 Info("StartAnalysis", "\n_______________________________________________________________________ \
3311 \n Running analysis script in a daughter shell as on a worker node \
3312 \n_______________________________________________________________________");
149d288c 3313 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3314 TIter next(list);
3315 TObjString *str;
a2f5fc01 3316 TString outputFile;
c57f56b7 3317 while((str=(TObjString*)next())) {
a2f5fc01 3318 outputFile = str->GetString();
3319 Int_t index = outputFile.Index("@");
3320 if (index > 0) outputFile.Remove(index);
3321 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3322 }
3323 delete list;
3324 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3325 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3326// gSystem->Exec("cat stdout");
5513444a 3327 return kFALSE;
c57f56b7 3328 }
5513444a 3329 // Check if submitting is managed by LPM manager
a3e84053 3330 if (fProductionMode) {
5513444a 3331 TString prodfile = fJDLName;
3332 prodfile.ReplaceAll(".jdl", ".prod");
3333 WriteProductionFile(prodfile);
3334 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3335 return kFALSE;
3336 }
a8739e8a 3337 // Submit AliEn job(s)
d2a409b2 3338 gGrid->Cd(fGridOutputDir);
a8739e8a 3339 TGridResult *res;
c57f56b7 3340 TString jobID = "";
d2a409b2 3341 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3342 // Submit a given xml or a set of runs
a8739e8a 3343 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3344 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3345 if (res) {
3346 const char *cjobId = res->GetKey(0,"jobId");
3347 if (!cjobId) {
a03be957 3348 gGrid->Stdout();
3349 gGrid->Stderr();
a8739e8a 3350 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3351 return kFALSE;
a8739e8a 3352 } else {
3353 Info("StartAnalysis", "\n_______________________________________________________________________ \
3354 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3355 \n_______________________________________________________________________",
3356 fJDLName.Data(), cjobId);
3357 jobID = cjobId;
3358 }
3359 delete res;
a03be957 3360 } else {
3361 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3362 return kFALSE;
a8739e8a 3363 }
3364 } else {
d2a409b2 3365 // Submit for a range of enumeration of runs.
a03be957 3366 if (!Submit()) return kFALSE;
c57f56b7 3367 }
a8739e8a 3368
c57f56b7 3369 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3370 \n You may exit at any time and terminate the job later using the option <terminate> \
3371 \n ##################################################################################", jobID.Data());
bb885a9e 3372 gSystem->Exec("aliensh");
5513444a 3373 return kTRUE;
c57f56b7 3374}
3375
f790bc1b 3376//______________________________________________________________________________
3377const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3378{
3379// Get a comma-separated list of output files of the requested type.
3380// Type can be (case unsensitive):
3381// aod - list of aod files (std, extensions and filters)
3382// out - list of output files connected to containers (but not aod's or extras)
3383// ext - list of extra files registered to the manager
3384// ter - list of files produced in terminate
3385 static TString files;
3386 files = "";
3387 TString stype = type;
3388 stype.ToLower();
3389 TString aodfiles, extra;
3390 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3391 if (!mgr) {
3392 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3393 return files.Data();
3394 }
3395 if (mgr->GetOutputEventHandler()) {
3396 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3397 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3398 if (!extraaod.IsNull()) {
3399 aodfiles += ",";
3400 aodfiles += extraaod;
3401 }
3402 }
3403 if (stype.Contains("aod")) {
3404 files = aodfiles;
3405 if (stype == "aod") return files.Data();
3406 }
3407 // Add output files that are not in the list of AOD files
3408 TString outputfiles = "";
3409 TIter next(mgr->GetOutputs());
3410 AliAnalysisDataContainer *output;
3411 const char *filename = 0;
3412 while ((output=(AliAnalysisDataContainer*)next())) {
3413 filename = output->GetFileName();
3414 if (!(strcmp(filename, "default"))) continue;
3415 if (outputfiles.Contains(filename)) continue;
3416 if (aodfiles.Contains(filename)) continue;
3417 if (!outputfiles.IsNull()) outputfiles += ",";
3418 outputfiles += filename;
3419 }
3420 if (stype.Contains("out")) {
3421 if (!files.IsNull()) files += ",";
3422 files += outputfiles;
3423 if (stype == "out") return files.Data();
3424 }
3425 // Add extra files registered to the analysis manager
3426 TString sextra;
3427 extra = mgr->GetExtraFiles();
3428 if (!extra.IsNull()) {
3429 extra.Strip();
3430 extra.ReplaceAll(" ", ",");
3431 TObjArray *fextra = extra.Tokenize(",");
3432 TIter nextx(fextra);
3433 TObject *obj;
3434 while ((obj=nextx())) {
3435 if (aodfiles.Contains(obj->GetName())) continue;
3436 if (outputfiles.Contains(obj->GetName())) continue;
3437 if (sextra.Contains(obj->GetName())) continue;
3438 if (!sextra.IsNull()) sextra += ",";
3439 sextra += obj->GetName();
3440 }
3441 delete fextra;
3442 if (stype.Contains("ext")) {
3443 if (!files.IsNull()) files += ",";
3444 files += sextra;
3445 }
3446 }
3447 if (stype == "ext") return files.Data();
3448 TString termfiles;
3449 if (!fTerminateFiles.IsNull()) {
3450 fTerminateFiles.Strip();
3451 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3452 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3453 TIter nextx(fextra);
3454 TObject *obj;
3455 while ((obj=nextx())) {
3456 if (aodfiles.Contains(obj->GetName())) continue;
3457 if (outputfiles.Contains(obj->GetName())) continue;
3458 if (termfiles.Contains(obj->GetName())) continue;
3459 if (sextra.Contains(obj->GetName())) continue;
3460 if (!termfiles.IsNull()) termfiles += ",";
3461 termfiles += obj->GetName();
3462 }
3463 delete fextra;
3464 }
3465 if (stype.Contains("ter")) {
3466 if (!files.IsNull() && !termfiles.IsNull()) {
3467 files += ",";
3468 files += termfiles;
3469 }
3470 }
3471 return files.Data();
3472}
3473
d2a409b2 3474//______________________________________________________________________________
a03be957 3475Bool_t AliAnalysisAlien::Submit()
d2a409b2 3476{
3477// Submit all master jobs.
3478 Int_t nmasterjobs = fInputFiles->GetEntries();
3479 Long_t tshoot = gSystem->Now();
a03be957 3480 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3481 while (fNsubmitted < nmasterjobs) {
3482 Long_t now = gSystem->Now();
3483 if ((now-tshoot)>30000) {
3484 tshoot = now;
a03be957 3485 if (!SubmitNext()) return kFALSE;
d2a409b2 3486 }
3487 }
a03be957 3488 return kTRUE;
d2a409b2 3489}
3490
0f389141 3491//______________________________________________________________________________
a03be957 3492Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3493{
3494// Submit all merging jobs.
dc030a24 3495 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3496 gGrid->Cd(fGridOutputDir);
3497 TString mergeJDLName = fExecutable;
3498 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3499 if (!fInputFiles) {
3500 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3501 return kFALSE;
3502 }
0f389141 3503 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3504 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3505 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3506 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3507 if (fOutputToRunNo) {
3508 // The output directory is the run number
3509 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3510 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3511 } else {
17b4e9a4 3512 if (!fRunNumbers.Length() && !fRunRange[0]) {
3513 // The output directory is the grid outdir
3514 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3515 runOutDir = fGridOutputDir;
3516 } else {
3517 // The output directory is the master number in 3 digits format
3518 printf("### Submitting merging job for master <%03d>\n", i);
3519 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3520 }
7c2cd90a 3521 }
3522 // Check now the number of merging stages.
2b222f7c 3523 TObjArray *list = fOutputFiles.Tokenize(",");
3524 TIter next(list);
3525 TObjString *str;
3526 TString outputFile;
3527 while((str=(TObjString*)next())) {
3528 outputFile = str->GetString();
3529 Int_t index = outputFile.Index("@");
3530 if (index > 0) outputFile.Remove(index);
37d21c01 3531 if (!fMergeExcludes.Contains(outputFile) &&
3532 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3533 }
3534 delete list;
b3e07543 3535 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3536 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3537 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3538 }
a03be957 3539 if (!ntosubmit) return kTRUE;
9f020952 3540 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3541 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3542 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3543 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3544 \n ################################################################################################################");
0f389141 3545 gSystem->Exec("aliensh");
a03be957 3546 return kTRUE;
0f389141 3547}
3548
d2a409b2 3549//______________________________________________________________________________
a03be957 3550Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3551{
08d5b699 3552// Submit next bunch of master jobs if the queue is free. The first master job is
3553// submitted right away, while the next will not be unless the previous was split.
3554// The plugin will not submit new master jobs if there are more that 500 jobs in
3555// waiting phase.
d2a409b2 3556 static Bool_t iscalled = kFALSE;
3557 static Int_t firstmaster = 0;
3558 static Int_t lastmaster = 0;
3559 static Int_t npermaster = 0;
a03be957 3560 if (iscalled) return kTRUE;
d2a409b2 3561 iscalled = kTRUE;
3562 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3563 Int_t ntosubmit = 0;
3564 TGridResult *res;
3565 TString jobID = "";
3bdcb562 3566 Int_t nmasterjobs = fInputFiles->GetEntries();
3567 if (!fNsubmitted) {
3568 ntosubmit = 1;
3569 if (!IsUseSubmitPolicy()) {
90d50a8c 3570 if (nmasterjobs>5)
3bdcb562 3571 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3572 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3573 ntosubmit = nmasterjobs;
3574 }
3575 } else {
d2a409b2 3576 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3577 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3578 // If last master not split, just return
a03be957 3579 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3580 // No more than 100 waiting jobs
08d5b699 3581 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3582 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3583 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3584 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3585 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3586 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3587 }
d2a409b2 3588 for (Int_t i=0; i<ntosubmit; i++) {
3589 // Submit for a range of enumeration of runs.
a03be957 3590 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3591 TString query;
cd11251e 3592 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3593 runOutDir.ReplaceAll(".xml", "");
3594 if (fOutputToRunNo)
3595 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3596 else
3597 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3598 printf("********* %s\n",query.Data());
d2a409b2 3599 res = gGrid->Command(query);
3600 if (res) {
98ca124f 3601 TString cjobId1 = res->GetKey(0,"jobId");
3602 if (!cjobId1.Length()) {
d2a409b2 3603 iscalled = kFALSE;
a03be957 3604 gGrid->Stdout();
3605 gGrid->Stderr();
3606 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3607 return kFALSE;
d2a409b2 3608 } else {
3609 Info("StartAnalysis", "\n_______________________________________________________________________ \
3610 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3611 \n_______________________________________________________________________",
98ca124f 3612 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3613 jobID += cjobId1;
3614 jobID += " ";
98ca124f 3615 lastmaster = cjobId1.Atoi();
d2a409b2 3616 if (!firstmaster) firstmaster = lastmaster;
3617 fNsubmitted++;
3618 }
3619 delete res;
a03be957 3620 } else {
3621 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3622 return kFALSE;
d2a409b2 3623 }
3624 }
3625 iscalled = kFALSE;
a03be957 3626 return kTRUE;
d2a409b2 3627}
3628
c57f56b7 3629//______________________________________________________________________________
3630void AliAnalysisAlien::WriteAnalysisFile()
3631{
f10e8481 3632// Write current analysis manager into the file <analysisFile>
3633 TString analysisFile = fExecutable;
3634 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3635 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3636 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3637 if (!mgr || !mgr->IsInitialized()) {
3638 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3639 return;
3640 }
3641 // Check analysis type
3642 TObject *handler;
3643 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3644 handler = (TObject*)mgr->GetInputEventHandler();
3645 if (handler) {
65eb22f8 3646 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3647 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3648 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3649 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3650 } else {
3651 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3652 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3653 }
c57f56b7 3654 }
3655 TDirectory *cdir = gDirectory;
f10e8481 3656 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3657 if (file) {
e16a394c 3658 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3659 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3660 // Unless merging makes no sense
3661 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3662 mgr->Write();
3663 delete file;
fe2d7fc2 3664 // Enable termination for local jobs
3665 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3666 }
3667 if (cdir) cdir->cd();
f10e8481 3668 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3669 }
3670 Bool_t copy = kTRUE;
d3b18c4c 3671 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3672 if (copy) {
3673 CdWork();
3674 TString workdir = gGrid->GetHomeDirectory();
3675 workdir += fGridWorkingDir;
d3b18c4c 3676 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3677 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3678 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3679 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3680 }
3681}
3682
3683//______________________________________________________________________________
3684void AliAnalysisAlien::WriteAnalysisMacro()
3685{
3686// Write the analysis macro that will steer the analysis in grid mode.
3687 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3688 ofstream out;
3689 out.open(fAnalysisMacro.Data(), ios::out);
3690 if (!out.good()) {
3691 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3692 return;
3693 }
5b273635 3694 Bool_t hasSTEERBase = kFALSE;
3695 Bool_t hasESD = kFALSE;
3696 Bool_t hasAOD = kFALSE;
3697 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3698 Bool_t hasOADB = kFALSE;
5b273635 3699 Bool_t hasANALYSISalice = kFALSE;
3700 Bool_t hasCORRFW = kFALSE;
c57f56b7 3701 TString func = fAnalysisMacro;
3702 TString type = "ESD";
3703 TString comment = "// Analysis using ";
c85cfc0f 3704 if (IsUseMCchain()) {
3705 type = "MC";
3706 comment += "MC";
3707 } else {
3708 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3709 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3710 type = "AOD";
3711 comment += "AOD";
3712 }
3713 }
0df6ccf2 3714 if (type!="AOD" && fFriendChainName!="") {
3715 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3716 return;
3717 }
c57f56b7 3718 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3719 else comment += " data";
3720 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3721 func.ReplaceAll(".C", "");
3722 out << "void " << func.Data() << "()" << endl;
3723 out << "{" << endl;
3724 out << comment.Data() << endl;
3725 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3726 out << " TStopwatch timer;" << endl;
3727 out << " timer.Start();" << endl << endl;
13f0c00b 3728 // Change temp directory to current one
3729 out << "// Set temporary merging directory to current one" << endl;
3730 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3731 out << "// Set temporary compilation directory to current one" << endl;
3732 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3733 // Reset existing include path
3734 out << "// Reset existing include path and add current directory first in the search" << endl;
3735 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3736 if (!fExecutableCommand.Contains("aliroot")) {
3737 out << "// load base root libraries" << endl;
3738 out << " gSystem->Load(\"libTree\");" << endl;
3739 out << " gSystem->Load(\"libGeom\");" << endl;
3740 out << " gSystem->Load(\"libVMC\");" << endl;
3741 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3742 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3743 }
d5c6455a 3744 if (fAdditionalRootLibs.Length()) {
47a4137d 3745 // in principle libtree /lib geom libvmc etc. can go into this list, too
3746 out << "// Add aditional libraries" << endl;
3747 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3748 TIter next(list);
3749 TObjString *str;
3750 while((str=(TObjString*)next())) {
3751 if (str->GetString().Contains(".so"))
3752 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3753 }
47a4137d 3754 if (list) delete list;
d5c6455a 3755 }
57377eb5 3756 out << "// Load analysis framework libraries" << endl;
652561d5 3757 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3758 if (!fPackages) {
f790bc1b 3759 if (!fExecutableCommand.Contains("aliroot")) {
3760 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3761 out << " gSystem->Load(\"libESD\");" << endl;
3762 out << " gSystem->Load(\"libAOD\");" << endl;
3763 }
4e5c5506 3764 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3765 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3766 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3767 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3768 } else {
4e5c5506 3769 TIter next(fPackages);
3770 TObject *obj;
57377eb5 3771 TString pkgname;
57377eb5 3772 while ((obj=next())) {
3773 pkgname = obj->GetName();
4478e6f1 3774 if (pkgname == "STEERBase" ||
3775 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3776 if (pkgname == "ESD" ||
3777 pkgname == "ESD.par") hasESD = kTRUE;
3778 if (pkgname == "AOD" ||
3779 pkgname == "AOD.par") hasAOD = kTRUE;
3780 if (pkgname == "ANALYSIS" ||
3781 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3782 if (pkgname == "OADB" ||
3783 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3784 if (pkgname == "ANALYSISalice" ||
3785 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3786 if (pkgname == "CORRFW" ||
3787 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3788 }
3789 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3790 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3791 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3792 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3793 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3794 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3795 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3796 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3797 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3798 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3799 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3800 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3801 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3802 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3803 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3804 out << "// Compile other par packages" << endl;
3805 next.Reset();
fcc9bb6f 3806 while ((obj=next())) {
3807 pkgname = obj->GetName();
4478e6f1 3808 if (pkgname == "STEERBase" ||
3809 pkgname == "STEERBase.par" ||
3810 pkgname == "ESD" ||
3811 pkgname == "ESD.par" ||
3812 pkgname == "AOD" ||
3813 pkgname == "AOD.par" ||
3814 pkgname == "ANALYSIS" ||
3815 pkgname == "ANALYSIS.par" ||
7ea40ac8 3816 pkgname == "OADB" ||
3817 pkgname == "OADB.par" ||
4478e6f1 3818 pkgname == "ANALYSISalice" ||
3819 pkgname == "ANALYSISalice.par" ||
3820 pkgname == "CORRFW" ||
3821 pkgname == "CORRFW.par") continue;
5b273635 3822 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3823 }
4e5c5506 3824 }
b3e07543 3825 out << "// include path" << endl;
3826 // Get the include path from the interpreter and remove entries pointing to AliRoot
3827 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3828 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3829 out << " TIter nextpath(listpaths);" << endl;
3830 out << " TObjString *pname;" << endl;
3831 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3832 out << " TString current = pname->GetName();" << endl;
3833 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3834 out << " gSystem->AddIncludePath(current);" << endl;
3835 out << " }" << endl;
3836 out << " if (listpaths) delete listpaths;" << endl;
3837 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3838 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3839 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3840 if (fAdditionalLibs.Length()) {
3841 out << "// Add aditional AliRoot libraries" << endl;
3842 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3843 TIter next(list);
3844 TObjString *str;
3845 while((str=(TObjString*)next())) {
3846 if (str->GetString().Contains(".so"))
3847 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3848 if (str->GetString().Contains(".par"))
3849 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3850 }
3851 if (list) delete list;
3852 }
3853 out << endl;
c57f56b7 3854 out << "// analysis source to be compiled at runtime (if any)" << endl;
3855 if (fAnalysisSource.Length()) {
3856 TObjArray *list = fAnalysisSource.Tokenize(" ");
3857 TIter next(list);
3858 TObjString *str;
3859 while((str=(TObjString*)next())) {
3860 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3861 }
3862 if (list) delete list;
3863 }
3864 out << endl;
b3e07543 3865// out << " printf(\"Currently load libraries:\\n\");" << endl;
3866// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3867 if (fFastReadOption) {
d3339be3 3868 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3869 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3870 out << "// fast xrootd reading enabled" << endl;
3871 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3872 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3873 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3874 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3875 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3876 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3877 }
3878 if (!IsLocalTest()) {
3879 out << "// connect to AliEn and make the chain" << endl;
3880 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3881 }
c57f56b7 3882 out << "// read the analysis manager from file" << endl;
f10e8481 3883 TString analysisFile = fExecutable;
3884 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3885 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3886 << analysisFile << "\");" << endl;
3887 out << " if (!mgr) return;" << endl;
21ca8e59 3888 if (IsLocalTest()) {
3889 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3890 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 3891 if (fFileForTestMode.IsNull())
3892 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3893 else
3894 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 3895 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
21ca8e59 3896 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 3897 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 3898 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 3899 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
3900 } else {
7a366e0b 3901 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 3902 out << " mgr->SetNSysInfo(100);" << endl;
3903 }
21ca8e59 3904 }
c57f56b7 3905 out << " mgr->PrintStatus();" << endl;
52b6a92b 3906 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3907 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3908 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3909 } else {
9a8288ff 3910 if (TestBit(AliAnalysisGrid::kTest))
3911 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3912 else
3913 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3914 }
3915 }
21ca8e59 3916 if (!IsLocalTest()) {
242accb2 3917 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3918 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3919 } else {
3920 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3921 }
f7498086 3922 out << " timer.Stop();" << endl;
3923 out << " timer.Print();" << endl;
c57f56b7 3924 out << "}" << endl << endl;
21ca8e59 3925 if (!IsLocalTest()) {
fcc9bb6f 3926 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3927 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3928 out << "{" << endl;
3929 out << "// Create a chain using url's from xml file" << endl;
242accb2 3930 out << " TString filename;" << endl;
3931 out << " Int_t run = 0;" << endl;
c85cfc0f 3932 if (IsUseMCchain()) {
3933 out << " TString treename = \"TE\";" << endl;
3934 } else {
3935 out << " TString treename = type;" << endl;
3936 out << " treename.ToLower();" << endl;
3937 out << " treename += \"Tree\";" << endl;
3938 }
e02fee64 3939 out << " printf(\"***************************************\\n\");" << endl;
3940 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3941 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3942 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3943 out << " if (!coll) {" << endl;
3944 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3945 out << " return NULL;" << endl;
3946 out << " }" << endl;
242accb2 3947 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3948 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3949 if(fFriendChainName!="") {
3950 out << " TChain *chainFriend = new TChain(treename);" << endl;
3951 }
c57f56b7 3952 out << " coll->Reset();" << endl;
0df6ccf2 3953 out << " while (coll->Next()) {" << endl;
242accb2 3954 out << " filename = coll->GetTURL("");" << endl;
3955 out << " if (mgr) {" << endl;
3956 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3957 out << " if (nrun && nrun != run) {" << endl;
3958 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3959 out << " mgr->SetRunFromPath(nrun);" << endl;
3960 out << " run = nrun;" << endl;
3961 out << " }" << endl;
3962 out << " }" << endl;
3963 out << " chain->Add(filename);" << endl;
0df6ccf2 3964 if(fFriendChainName!="") {
3965 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3966 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3967 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3968 out << " chainFriend->Add(fileFriend.Data());" << endl;
3969 }
3970 out << " }" << endl;
c57f56b7 3971 out << " if (!chain->GetNtrees()) {" << endl;
3972 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3973 out << " return NULL;" << endl;
3974 out << " }" << endl;
0df6ccf2 3975 if(fFriendChainName!="") {
3976 out << " chain->AddFriend(chainFriend);" << endl;
3977 }
c57f56b7 3978 out << " return chain;" << endl;
fcc9bb6f 3979 out << "}" << endl << endl;
c57f56b7 3980 }
5b273635 3981 if (hasANALYSISalice) {
3982 out <<"//________________________________________________________________________________" << endl;
3983 out << "Bool_t SetupPar(const char *package) {" << endl;
3984 out << "// Compile the package and set it up." << endl;
3985 out << " TString pkgdir = package;" << endl;
3986 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3987 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3988 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3989 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3990 out << " // Check for BUILD.sh and execute" << endl;
3991 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3992 out << " printf(\"*******************************\\n\");" << endl;
3993 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3994 out << " printf(\"*******************************\\n\");" << endl;
3995 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3996 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3997 out << " gSystem->ChangeDirectory(cdir);" << endl;
3998 out << " return kFALSE;" << endl;
3999 out << " }" << endl;
4000 out << " } else {" << endl;
4001 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4002 out << " gSystem->ChangeDirectory(cdir);" << endl;
4003 out << " return kFALSE;" << endl;
4004 out << " }" << endl;
4005 out << " // Check for SETUP.C and execute" << endl;
4006 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4007 out << " printf(\"*******************************\\n\");" << endl;
4008 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4009 out << " printf(\"*******************************\\n\");" << endl;
4010 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4011 out << " } else {" << endl;
4012 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4013 out << " gSystem->ChangeDirectory(cdir);" << endl;
4014 out << " return kFALSE;" << endl;
4015 out << " }" << endl;
4016 out << " // Restore original workdir" << endl;
4017 out << " gSystem->ChangeDirectory(cdir);" << endl;
4018 out << " return kTRUE;" << endl;
4019 out << "}" << endl;
4020 }
c57f56b7 4021 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4022 }
4023 Bool_t copy = kTRUE;
d3b18c4c 4024 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4025 if (copy) {
4026 CdWork();
4027 TString workdir = gGrid->GetHomeDirectory();
4028 workdir += fGridWorkingDir;
4029 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4030 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4031// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4032 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4033 Form("alien://%s/%s", workdir.Data(),
4034 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4035 }
4036}
4037
0f389141 4038//______________________________________________________________________________
4039void AliAnalysisAlien::WriteMergingMacro()
4040{
4041// Write a macro to merge the outputs per master job.
4042 if (!fMergeViaJDL) return;
4043 if (!fOutputFiles.Length()) {
4044 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4045 return;
4046 }
4047 TString mergingMacro = fExecutable;
4048 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4049 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4050 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4051 ofstream out;
4052 out.open(mergingMacro.Data(), ios::out);
4053 if (!out.good()) {
4054 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4055 return;
4056 }
5b273635 4057 Bool_t hasSTEERBase = kFALSE;
4058 Bool_t hasESD = kFALSE;
4059 Bool_t hasAOD = kFALSE;
4060 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4061 Bool_t hasOADB = kFALSE;
5b273635 4062 Bool_t hasANALYSISalice = kFALSE;
4063 Bool_t hasCORRFW = kFALSE;
0f389141 4064 TString func = mergingMacro;
4065 TString comment;
4066 func.ReplaceAll(".C", "");
f47d5cb4 4067 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4068 out << "{" << endl;
4069 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4070 out << " TStopwatch timer;" << endl;
4071 out << " timer.Start();" << endl << endl;
b3e07543 4072 // Reset existing include path
4073 out << "// Reset existing include path and add current directory first in the search" << endl;
4074 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4075 if (!fExecutableCommand.Contains("aliroot")) {
4076 out << "// load base root libraries" << endl;
4077 out << " gSystem->Load(\"libTree\");" << endl;
4078 out << " gSystem->Load(\"libGeom\");" << endl;
4079 out << " gSystem->Load(\"libVMC\");" << endl;
4080 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4081 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4082 }
0f389141 4083 if (fAdditionalRootLibs.Length()) {
4084 // in principle libtree /lib geom libvmc etc. can go into this list, too
4085 out << "// Add aditional libraries" << endl;
4086 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4087 TIter next(list);
4088 TObjString *str;
4089 while((str=(TObjString*)next())) {
4090 if (str->GetString().Contains(".so"))
4091 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4092 }
4093 if (list) delete list;
4094 }
0f389141 4095 out << "// Load analysis framework libraries" << endl;
4096 if (!fPackages) {
7c2cd90a 4097 if (!fExecutableCommand.Contains("aliroot")) {
4098 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4099 out << " gSystem->Load(\"libESD\");" << endl;
4100 out << " gSystem->Load(\"libAOD\");" << endl;
4101 }
0f389141 4102 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4103 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4104 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4105 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4106 } else {
4107 TIter next(fPackages);
4108 TObject *obj;
4109 TString pkgname;
5b273635 4110 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4111 while ((obj=next())) {
4112 pkgname = obj->GetName();
4113 if (pkgname == "STEERBase" ||
4114 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4115 if (pkgname == "ESD" ||
4116 pkgname == "ESD.par") hasESD = kTRUE;
4117 if (pkgname == "AOD" ||
4118 pkgname == "AOD.par") hasAOD = kTRUE;
4119 if (pkgname == "ANALYSIS" ||
4120 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4121 if (pkgname == "OADB" ||
4122 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4123 if (pkgname == "ANALYSISalice" ||
4124 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4125 if (pkgname == "CORRFW" ||
4126 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4127 }
5b273635 4128 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4129 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4130 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4131 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4132 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4133 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4134 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4135 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4136 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4137 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4138 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4139 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4140 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4141 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4142 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4143 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4144 out << "// Compile other par packages" << endl;
4145 next.Reset();
4146 while ((obj=next())) {
4147 pkgname = obj->GetName();
4148 if (pkgname == "STEERBase" ||
4149 pkgname == "STEERBase.par" ||
4150 pkgname == "ESD" ||
4151 pkgname == "ESD.par" ||
4152 pkgname == "AOD" ||
4153 pkgname == "AOD.par" ||
4154 pkgname == "ANALYSIS" ||
4155 pkgname == "ANALYSIS.par" ||
7ea40ac8 4156 pkgname == "OADB" ||
4157 pkgname == "OADB.par" ||
0f389141 4158 pkgname == "ANALYSISalice" ||
4159 pkgname == "ANALYSISalice.par" ||
4160 pkgname == "CORRFW" ||
4161 pkgname == "CORRFW.par") continue;
5b273635 4162 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4163 }
4164 }
b3e07543 4165 out << "// include path" << endl;
4166 // Get the include path from the interpreter and remove entries pointing to AliRoot
4167 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4168 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4169 out << " TIter nextpath(listpaths);" << endl;
4170 out << " TObjString *pname;" << endl;
4171 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4172 out << " TString current = pname->GetName();" << endl;
4173 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4174 out << " gSystem->AddIncludePath(current);" << endl;
4175 out << " }" << endl;
4176 out << " if (listpaths) delete listpaths;" << endl;
4177 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4178 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4179 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4180 if (fAdditionalLibs.Length()) {
4181 out << "// Add aditional AliRoot libraries" << endl;
4182 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4183 TIter next(list);
4184 TObjString *str;
4185 while((str=(TObjString*)next())) {
4186 if (str->GetString().Contains(".so"))
4187 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4188 }
4189 if (list) delete list;
4190 }
4191 out << endl;
4192 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4193 if (fAnalysisSource.Length()) {
4194 TObjArray *list = fAnalysisSource.Tokenize(" ");
4195 TIter next(list);
4196 TObjString *str;
4197 while((str=(TObjString*)next())) {
4198 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4199 }
4200 if (list) delete list;
4201 }
149d288c 4202 out << endl;
4203
0f389141 4204 if (fFastReadOption) {
4205 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4206 out << "// fast xrootd reading enabled" << endl;
4207 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4208 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4209 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4210 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4211 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4212 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4213 }
4214 // Change temp directory to current one
4215 out << "// Set temporary merging directory to current one" << endl;
4216 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4217 out << "// Set temporary compilation directory to current one" << endl;
4218 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4219 out << "// Connect to AliEn" << endl;
4220 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4221 out << " TString outputDir = dir;" << endl;
f790bc1b 4222 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4223 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4224 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4225 out << " TIter *iter = new TIter(list);" << endl;
4226 out << " TObjString *str;" << endl;
a2f5fc01 4227 out << " TString outputFile;" << endl;
0f389141 4228 out << " Bool_t merged = kTRUE;" << endl;
4229 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4230 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4231 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4232 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4233 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4234 out << " // Skip already merged outputs" << endl;
a2f5fc01 4235 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4236 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4237 out << " continue;" << endl;
4238 out << " }" << endl;
a2f5fc01 4239 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4240 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4241 out << " if (!merged) {" << endl;
a2f5fc01 4242 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4243 out << " return;" << endl;
0f389141 4244 out << " }" << endl;
4245 out << " }" << endl;
7c2cd90a 4246 out << " // all outputs merged, validate" << endl;
4247 out << " ofstream out;" << endl;
4248 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4249 out << " out.close();" << endl;
4250 out << " // read the analysis manager from file" << endl;
f866cba5 4251 TString analysisFile = fExecutable;
4252 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4253 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4254 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4255 << analysisFile << "\");" << endl;
4256 out << " if (!mgr) return;" << endl;
5c4250fc 4257 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4258 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4259 out << " mgr->PrintStatus();" << endl;
4260 if (AliAnalysisManager::GetAnalysisManager()) {
4261 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4262 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4263 } else {
9a8288ff 4264 if (TestBit(AliAnalysisGrid::kTest))
4265 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4266 else
4267 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4268 }
4269 }
9a8288ff 4270 out << " TTree *tree = NULL;" << endl;
4271 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4272 out << "}" << endl << endl;
5b273635 4273 if (hasANALYSISalice) {
4274 out <<"//________________________________________________________________________________" << endl;
4275 out << "Bool_t SetupPar(const char *package) {" << endl;
4276 out << "// Compile the package and set it up." << endl;
4277 out << " TString pkgdir = package;" << endl;
4278 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4279 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4280 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4281 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4282 out << " // Check for BUILD.sh and execute" << endl;
4283 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4284 out << " printf(\"*******************************\\n\");" << endl;
4285 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4286 out << " printf(\"*******************************\\n\");" << endl;
4287 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4288 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4289 out << " gSystem->ChangeDirectory(cdir);" << endl;
4290 out << " return kFALSE;" << endl;
4291 out << " }" << endl;
4292 out << " } else {" << endl;
4293 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4294 out << " gSystem->ChangeDirectory(cdir);" << endl;
4295 out << " return kFALSE;" << endl;
4296 out << " }" << endl;
4297 out << " // Check for SETUP.C and execute" << endl;
4298 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4299 out << " printf(\"*******************************\\n\");" << endl;
4300 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4301 out << " printf(\"*******************************\\n\");" << endl;
4302 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4303 out << " } else {" << endl;
4304 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4305 out << " gSystem->ChangeDirectory(cdir);" << endl;
4306 out << " return kFALSE;" << endl;
4307 out << " }" << endl;
4308 out << " // Restore original workdir" << endl;
4309 out << " gSystem->ChangeDirectory(cdir);" << endl;
4310 out << " return kTRUE;" << endl;
4311 out << "}" << endl;
4312 }
0f389141 4313 }
4314 Bool_t copy = kTRUE;
d3b18c4c 4315 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4316 if (copy) {
4317 CdWork();
4318 TString workdir = gGrid->GetHomeDirectory();
4319 workdir += fGridWorkingDir;
4320 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4321 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4322// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4323 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4324 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4325 }
4326}
4327
4328//______________________________________________________________________________
4329Bool_t AliAnalysisAlien::SetupPar(const char *package)
4330{
205b201f 4331// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4332// Note that for loading the compiled library. The current directory should have precedence in
4333// LD_LIBRARY_PATH
4334 TString pkgdir = package;
4335 pkgdir.ReplaceAll(".par","");
b50a41c3 4336 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4337 TString cdir = gSystem->WorkingDirectory();
4338 gSystem->ChangeDirectory(pkgdir);
4339 // Check for BUILD.sh and execute
4340 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4341 printf("**************************************************\n");
4342 printf("*** Building PAR archive %s\n", package);
4343 printf("**************************************************\n");
4344 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4345 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4346 gSystem->ChangeDirectory(cdir);
4347 return kFALSE;
4348 }
4349 } else {
4350 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4351 gSystem->ChangeDirectory(cdir);
4352 return kFALSE;
4353 }
4354 // Check for SETUP.C and execute
4355 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4356 printf("**************************************************\n");
4357 printf("*** Setup PAR archive %s\n", package);
4358 printf("**************************************************\n");
4359 gROOT->Macro("PROOF-INF/SETUP.C");
4360 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4361 } else {
4362 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4363 gSystem->ChangeDirectory(cdir);
4364 return kFALSE;
4365 }
4366 // Restore original workdir
4367 gSystem->ChangeDirectory(cdir);
4368 return kTRUE;
4369}
4370
c57f56b7 4371//______________________________________________________________________________
4372void AliAnalysisAlien::WriteExecutable()
4373{
4374// Generate the alien executable script.
4375 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4376 ofstream out;
4377 out.open(fExecutable.Data(), ios::out);
4378 if (out.bad()) {
5513444a 4379 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4380 return;
4381 }
4382 out << "#!/bin/bash" << endl;
2a4d5166 4383 // Make sure we can properly compile par files
4384 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4385 out << "echo \"=========================================\"" << endl;
4386 out << "echo \"############## PATH : ##############\"" << endl;
4387 out << "echo $PATH" << endl;
4388 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4389 out << "echo $LD_LIBRARY_PATH" << endl;
4390 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4391 out << "echo $ROOTSYS" << endl;
4392 out << "echo \"############## which root : ##############\"" << endl;
4393 out << "which root" << endl;
4394 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4395 out << "echo $ALICE_ROOT" << endl;
4396 out << "echo \"############## which aliroot : ##############\"" << endl;
4397 out << "which aliroot" << endl;
9c5ddadc 4398 out << "echo \"############## system limits : ##############\"" << endl;
4399 out << "ulimit -a" << endl;
4400 out << "echo \"############## memory : ##############\"" << endl;
4401 out << "free -m" << endl;
c57f56b7 4402 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4403 out << fExecutableCommand << " ";
370c9880 4404 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4405 out << "RET=$?" << endl;
4406 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4407 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4408 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4409 out << " let sig=\"$RET - 128\""<<endl;
4410 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4411 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4412 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4413 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4414 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4415 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4416 out << " fi"<<endl;
4417 out << " exit $RET"<< endl;
4418 out << "fi" << endl << endl ;
4419 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4420 out << "echo \"############## memory after: ##############\"" << endl;
4421 out << "free -m" << endl;
c57f56b7 4422 }
4423 Bool_t copy = kTRUE;
d3b18c4c 4424 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4425 if (copy) {
4426 CdWork();
4427 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4428 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4429 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4430 workdir += fGridWorkingDir;
4431 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4432 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4433 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4434// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4435 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4436 executable.Data())) Fatal("","Terminating");
c57f56b7 4437 }
4438}
4439
0f389141 4440//______________________________________________________________________________
4441void AliAnalysisAlien::WriteMergeExecutable()
4442{
4443// Generate the alien executable script for the merging job.
4444 if (!fMergeViaJDL) return;
4445 TString mergeExec = fExecutable;
4446 mergeExec.ReplaceAll(".sh", "_merge.sh");
4447 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4448 ofstream out;
4449 out.open(mergeExec.Data(), ios::out);
4450 if (out.bad()) {
4451 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4452 return;
4453 }
4454 out << "#!/bin/bash" << endl;
2a4d5166 4455 // Make sure we can properly compile par files
4456 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4457 out << "echo \"=========================================\"" << endl;
4458 out << "echo \"############## PATH : ##############\"" << endl;
4459 out << "echo $PATH" << endl;
4460 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4461 out << "echo $LD_LIBRARY_PATH" << endl;
4462 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4463 out << "echo $ROOTSYS" << endl;
4464 out << "echo \"############## which root : ##############\"" << endl;
4465 out << "which root" << endl;
4466 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4467 out << "echo $ALICE_ROOT" << endl;
4468 out << "echo \"############## which aliroot : ##############\"" << endl;
4469 out << "which aliroot" << endl;
4470 out << "echo \"############## system limits : ##############\"" << endl;
4471 out << "ulimit -a" << endl;
4472 out << "echo \"############## memory : ##############\"" << endl;
4473 out << "free -m" << endl;
4474 out << "echo \"=========================================\"" << endl << endl;
0f389141 4475 TString mergeMacro = fExecutable;
4476 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4477 if (IsOneStageMerging())
4478 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4479 else
f47d5cb4 4480 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4481 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4482 out << "RET=$?" << endl;
4483 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4484 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4485 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4486 out << " let sig=\"$RET - 128\""<<endl;
4487 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4488 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4489 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4490 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4491 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4492 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4493 out << " fi"<<endl;
4494 out << " exit $RET"<< endl;
4495 out << "fi" << endl << endl ;
0f389141 4496 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4497 out << "echo \"############## memory after: ##############\"" << endl;
4498 out << "free -m" << endl;
0f389141 4499 }
4500 Bool_t copy = kTRUE;
d3b18c4c 4501 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4502 if (copy) {
4503 CdWork();
4504 TString workdir = gGrid->GetHomeDirectory();
4505 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4506 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4507 workdir += fGridWorkingDir;
4508 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4509 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4510 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4511// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4512 if (!copyLocal2Alien("WriteMergeExecutable",
4513 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4514 }
4515}
4516
c57f56b7 4517//______________________________________________________________________________
5513444a 4518void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4519{
4520// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4521// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4522// Next lines: full_path_to_dataset XXX (XXX is a string)
4523// To submit, one has to: submit jdl XXX for all lines
4524 ofstream out;
4525 out.open(filename, ios::out);
4526 if (out.bad()) {
4527 Error("WriteProductionFile", "Bad file name: %s", filename);
4528 return;
4529 }
d3b18c4c 4530 TString workdir;
4531 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4532 workdir = gGrid->GetHomeDirectory();
5513444a 4533 workdir += fGridWorkingDir;
f5e8c702 4534 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4535 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4536 out << locjdl << " " << njobspermaster << endl;
5513444a 4537 Int_t nmasterjobs = fInputFiles->GetEntries();
4538 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4539 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4540 runOutDir.ReplaceAll(".xml", "");
4541 if (fOutputToRunNo)
4542 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4543 else
4544 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4545 }
d3b18c4c 4546 if (gGrid) {
4547 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4548 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4549// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4550 if (!copyLocal2Alien("WriteProductionFile", filename,
4551 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4552 }
5513444a 4553}
4554
4555//______________________________________________________________________________
0f389141 4556void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4557{
4558// Generate the alien validation script.
4559 // Generate the validation script
4560 TObjString *os;
d3b18c4c 4561 if (fValidationScript.IsNull()) {
4562 fValidationScript = fExecutable;
4563 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4564 }
4565 TString validationScript = fValidationScript;
4566 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4567 if (!Connect()) {
4568 Error("WriteValidationScript", "Alien connection required");
4569 return;
4570 }
d3b18c4c 4571 if (!fTerminateFiles.IsNull()) {
4572 fTerminateFiles.Strip();
4573 fTerminateFiles.ReplaceAll(" ",",");
4574 }
a2f5fc01 4575 TString outStream = "";
4576 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4577 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4578 ofstream out;
0d5d317c 4579 out.open(validationScript, ios::out);
c57f56b7 4580 out << "#!/bin/bash" << endl;
4581 out << "##################################################" << endl;
4582 out << "validateout=`dirname $0`" << endl;
4583 out << "validatetime=`date`" << endl;
4584 out << "validated=\"0\";" << endl;
4585 out << "error=0" << endl;
4586 out << "if [ -z $validateout ]" << endl;
4587 out << "then" << endl;
4588 out << " validateout=\".\"" << endl;
4589 out << "fi" << endl << endl;
4590 out << "cd $validateout;" << endl;
4591 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4592 out << "echo \"*******************************************************\"" << outStream << endl;
4593 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4594 out << "" << endl;
a2f5fc01 4595 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4596 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4597 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4598 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4599 out << "ls -la ./" << outStream << endl;
4600 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4601 out << "##################################################" << endl;
ebec370a 4602 out << "" << endl;
4603
4604 out << "if [ ! -f stderr ] ; then" << endl;
4605 out << " error=1" << endl;
a2f5fc01 4606 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4607 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4608 out << "fi" << endl;
4609
b34c9f51 4610 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4611 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4612 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4613 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4614 out << "" << endl;
4615
ebec370a 4616 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4617 out << " error=1" << endl;
a2f5fc01 4618 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4619 out << " echo \"$parArch\" " << outStream << endl;
4620 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4621 out << "fi" << endl;
4622
4623 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4624 out << " error=1" << endl;
a2f5fc01 4625 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4626 out << " echo \"$segViol\" " << outStream << endl;
4627 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4628 out << "fi" << endl;
4629
4630 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4631 out << " error=1" << endl;
a2f5fc01 4632 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4633 out << " echo \"$segFault\" " << outStream << endl;
4634 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4635 out << "fi" << endl;
4636
b34c9f51 4637 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4638 out << " error=1" << endl;
a2f5fc01 4639 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4640 out << " echo \"$glibcErr\" " << outStream << endl;
4641 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4642 out << "fi" << endl;
4643
ebec370a 4644 // Part dedicated to the specific analyses running into the train
4645
d3b18c4c 4646 TString outputFiles = fOutputFiles;
4647 if (merge && !fTerminateFiles.IsNull()) {
4648 outputFiles += ",";
4649 outputFiles += fTerminateFiles;
4650 }
4651 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4652 TIter next1(arr);
a2f5fc01 4653 TString outputFile;
bbd88db8 4654 while (!merge && (os=(TObjString*)next1())) {
4655 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4656 outputFile = os->GetString();
4657 Int_t index = outputFile.Index("@");
4658 if (index > 0) outputFile.Remove(index);
bbd88db8 4659 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4660 if (outputFile.Contains("*")) continue;
4661 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4662 out << " error=1" << endl;
7c2cd90a 4663 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4664 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4665 out << "fi" << endl;
4666 }
4667 delete arr;
7c2cd90a 4668 out << "if ! [ -f outputs_valid ] ; then" << endl;
4669 out << " error=1" << endl;
4670 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4671 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4672 out << "fi" << endl;
923e2ca5 4673
c57f56b7 4674 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4675 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4676 if (!IsKeepLogs()) {
4677 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4678 outStream = "";
149d288c 4679 out << " rm -f std*" << endl;
4680 }
c57f56b7 4681 out << "fi" << endl;
4682
a2f5fc01 4683 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4684 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4685 out << "cd -" << endl;
4686 out << "exit $error" << endl;
4687 }
4688 Bool_t copy = kTRUE;
d3b18c4c 4689 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4690 if (copy) {
4691 CdWork();
4692 TString workdir = gGrid->GetHomeDirectory();
4693 workdir += fGridWorkingDir;
d3b18c4c 4694 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4695 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4696// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4697 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4698 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4699 }
4700}