]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Peakfinder unit added to compilation
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
f965131e 120 fIncludePath(),
bb885a9e 121 fCloseSE(),
0df6ccf2 122 fFriendChainName(),
c6cb3634 123 fJobTag(),
648174cf 124 fOutputSingle(),
5fce53f4 125 fRunPrefix(),
3bdcb562 126 fProofCluster(),
127 fProofDataSet(),
128 fFileForTestMode(),
129 fRootVersionForProof(),
130 fAliRootMode(),
f47d5cb4 131 fMergeDirName(),
4e5c5506 132 fInputFiles(0),
28ce6c9c 133 fPackages(0),
77f34eae 134 fModules(0),
28ce6c9c 135 fProofParam()
c57f56b7 136{
137// Dummy ctor.
138 SetDefaults();
139}
140
141//______________________________________________________________________________
142AliAnalysisAlien::AliAnalysisAlien(const char *name)
143 :AliAnalysisGrid(name),
144 fGridJDL(NULL),
0f389141 145 fMergingJDL(NULL),
c57f56b7 146 fPrice(0),
147 fTTL(0),
148 fSplitMaxInputFileNumber(0),
149 fMaxInitFailed(0),
150 fMasterResubmitThreshold(0),
bb885a9e 151 fNtestFiles(0),
319593fb 152 fNrunsPerMaster(0),
16a4353c 153 fMaxMergeFiles(0),
b3e07543 154 fMaxMergeStages(0),
d2a409b2 155 fNsubmitted(0),
a3e84053 156 fProductionMode(0),
cd11251e 157 fOutputToRunNo(0),
0f389141 158 fMergeViaJDL(0),
159 fFastReadOption(0),
e1c22e21 160 fOverwriteMode(1),
149d288c 161 fNreplicas(2),
3bdcb562 162 fNproofWorkers(0),
27734f0e 163 fNproofWorkersPerSlave(0),
3bdcb562 164 fProofReset(0),
c57f56b7 165 fRunNumbers(),
166 fExecutable(),
0a1c1f7f 167 fExecutableCommand(),
c57f56b7 168 fArguments(),
631c0b05 169 fExecutableArgs(),
c57f56b7 170 fAnalysisMacro(),
171 fAnalysisSource(),
d3b18c4c 172 fValidationScript(),
d5c6455a 173 fAdditionalRootLibs(),
c57f56b7 174 fAdditionalLibs(),
175 fSplitMode(),
176 fAPIVersion(),
177 fROOTVersion(),
178 fAliROOTVersion(),
648174cf 179 fExternalPackages(),
c57f56b7 180 fUser(),
181 fGridWorkingDir(),
182 fGridDataDir(),
183 fDataPattern(),
184 fGridOutputDir(),
185 fOutputArchive(),
186 fOutputFiles(),
187 fInputFormat(),
e7c71df0 188 fDatasetName(),
c57f56b7 189 fJDLName(),
d3b18c4c 190 fTerminateFiles(),
bb885a9e 191 fMergeExcludes(),
f965131e 192 fIncludePath(),
bb885a9e 193 fCloseSE(),
0df6ccf2 194 fFriendChainName(),
c6cb3634 195 fJobTag(),
648174cf 196 fOutputSingle(),
5fce53f4 197 fRunPrefix(),
3bdcb562 198 fProofCluster(),
199 fProofDataSet(),
200 fFileForTestMode(),
201 fRootVersionForProof(),
202 fAliRootMode(),
f47d5cb4 203 fMergeDirName(),
4e5c5506 204 fInputFiles(0),
28ce6c9c 205 fPackages(0),
77f34eae 206 fModules(0),
28ce6c9c 207 fProofParam()
c57f56b7 208{
209// Default ctor.
210 SetDefaults();
211}
212
213//______________________________________________________________________________
214AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
215 :AliAnalysisGrid(other),
216 fGridJDL(NULL),
0f389141 217 fMergingJDL(NULL),
c57f56b7 218 fPrice(other.fPrice),
219 fTTL(other.fTTL),
220 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
221 fMaxInitFailed(other.fMaxInitFailed),
222 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 223 fNtestFiles(other.fNtestFiles),
319593fb 224 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 225 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 226 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 227 fNsubmitted(other.fNsubmitted),
a3e84053 228 fProductionMode(other.fProductionMode),
cd11251e 229 fOutputToRunNo(other.fOutputToRunNo),
0f389141 230 fMergeViaJDL(other.fMergeViaJDL),
231 fFastReadOption(other.fFastReadOption),
232 fOverwriteMode(other.fOverwriteMode),
149d288c 233 fNreplicas(other.fNreplicas),
3bdcb562 234 fNproofWorkers(other.fNproofWorkers),
27734f0e 235 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 236 fProofReset(other.fProofReset),
c57f56b7 237 fRunNumbers(other.fRunNumbers),
238 fExecutable(other.fExecutable),
0a1c1f7f 239 fExecutableCommand(other.fExecutableCommand),
c57f56b7 240 fArguments(other.fArguments),
631c0b05 241 fExecutableArgs(other.fExecutableArgs),
c57f56b7 242 fAnalysisMacro(other.fAnalysisMacro),
243 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 244 fValidationScript(other.fValidationScript),
d5c6455a 245 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 246 fAdditionalLibs(other.fAdditionalLibs),
247 fSplitMode(other.fSplitMode),
248 fAPIVersion(other.fAPIVersion),
249 fROOTVersion(other.fROOTVersion),
250 fAliROOTVersion(other.fAliROOTVersion),
648174cf 251 fExternalPackages(other.fExternalPackages),
c57f56b7 252 fUser(other.fUser),
253 fGridWorkingDir(other.fGridWorkingDir),
254 fGridDataDir(other.fGridDataDir),
255 fDataPattern(other.fDataPattern),
256 fGridOutputDir(other.fGridOutputDir),
257 fOutputArchive(other.fOutputArchive),
258 fOutputFiles(other.fOutputFiles),
259 fInputFormat(other.fInputFormat),
e7c71df0 260 fDatasetName(other.fDatasetName),
c57f56b7 261 fJDLName(other.fJDLName),
d3b18c4c 262 fTerminateFiles(other.fTerminateFiles),
bb885a9e 263 fMergeExcludes(other.fMergeExcludes),
f965131e 264 fIncludePath(other.fIncludePath),
bb885a9e 265 fCloseSE(other.fCloseSE),
0df6ccf2 266 fFriendChainName(other.fFriendChainName),
c6cb3634 267 fJobTag(other.fJobTag),
648174cf 268 fOutputSingle(other.fOutputSingle),
5fce53f4 269 fRunPrefix(other.fRunPrefix),
3bdcb562 270 fProofCluster(other.fProofCluster),
271 fProofDataSet(other.fProofDataSet),
272 fFileForTestMode(other.fFileForTestMode),
273 fRootVersionForProof(other.fRootVersionForProof),
274 fAliRootMode(other.fAliRootMode),
f47d5cb4 275 fMergeDirName(other.fMergeDirName),
4e5c5506 276 fInputFiles(0),
28ce6c9c 277 fPackages(0),
77f34eae 278 fModules(0),
28ce6c9c 279 fProofParam()
c57f56b7 280{
281// Copy ctor.
282 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 283 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 284 fRunRange[0] = other.fRunRange[0];
285 fRunRange[1] = other.fRunRange[1];
c57f56b7 286 if (other.fInputFiles) {
287 fInputFiles = new TObjArray();
288 TIter next(other.fInputFiles);
289 TObject *obj;
290 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
291 fInputFiles->SetOwner();
292 }
4e5c5506 293 if (other.fPackages) {
294 fPackages = new TObjArray();
295 TIter next(other.fPackages);
296 TObject *obj;
297 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
298 fPackages->SetOwner();
299 }
77f34eae 300 if (other.fModules) {
301 fModules = new TObjArray();
302 fModules->SetOwner();
303 TIter next(other.fModules);
304 AliAnalysisTaskCfg *mod, *crt;
305 while ((crt=(AliAnalysisTaskCfg*)next())) {
306 mod = new AliAnalysisTaskCfg(*crt);
307 fModules->Add(mod);
308 }
309 }
c57f56b7 310}
311
312//______________________________________________________________________________
313AliAnalysisAlien::~AliAnalysisAlien()
314{
315// Destructor.
77f34eae 316 delete fGridJDL;
317 delete fMergingJDL;
318 delete fInputFiles;
319 delete fPackages;
320 delete fModules;
28ce6c9c 321 fProofParam.DeleteAll();
c57f56b7 322}
323
324//______________________________________________________________________________
325AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
326{
327// Assignment.
328 if (this != &other) {
329 AliAnalysisGrid::operator=(other);
330 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 331 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 332 fPrice = other.fPrice;
333 fTTL = other.fTTL;
334 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
335 fMaxInitFailed = other.fMaxInitFailed;
336 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 337 fNtestFiles = other.fNtestFiles;
a3e84053 338 fNrunsPerMaster = other.fNrunsPerMaster;
339 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 340 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 341 fNsubmitted = other.fNsubmitted;
342 fProductionMode = other.fProductionMode;
cd11251e 343 fOutputToRunNo = other.fOutputToRunNo;
0f389141 344 fMergeViaJDL = other.fMergeViaJDL;
345 fFastReadOption = other.fFastReadOption;
346 fOverwriteMode = other.fOverwriteMode;
149d288c 347 fNreplicas = other.fNreplicas;
3bdcb562 348 fNproofWorkers = other.fNproofWorkers;
27734f0e 349 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 350 fProofReset = other.fProofReset;
c57f56b7 351 fRunNumbers = other.fRunNumbers;
352 fExecutable = other.fExecutable;
0a1c1f7f 353 fExecutableCommand = other.fExecutableCommand;
c57f56b7 354 fArguments = other.fArguments;
631c0b05 355 fExecutableArgs = other.fExecutableArgs;
c57f56b7 356 fAnalysisMacro = other.fAnalysisMacro;
357 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 358 fValidationScript = other.fValidationScript;
d5c6455a 359 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 360 fAdditionalLibs = other.fAdditionalLibs;
361 fSplitMode = other.fSplitMode;
362 fAPIVersion = other.fAPIVersion;
363 fROOTVersion = other.fROOTVersion;
364 fAliROOTVersion = other.fAliROOTVersion;
648174cf 365 fExternalPackages = other.fExternalPackages;
c57f56b7 366 fUser = other.fUser;
367 fGridWorkingDir = other.fGridWorkingDir;
368 fGridDataDir = other.fGridDataDir;
369 fDataPattern = other.fDataPattern;
370 fGridOutputDir = other.fGridOutputDir;
371 fOutputArchive = other.fOutputArchive;
372 fOutputFiles = other.fOutputFiles;
373 fInputFormat = other.fInputFormat;
e7c71df0 374 fDatasetName = other.fDatasetName;
c57f56b7 375 fJDLName = other.fJDLName;
d3b18c4c 376 fTerminateFiles = other.fTerminateFiles;
bb885a9e 377 fMergeExcludes = other.fMergeExcludes;
f965131e 378 fIncludePath = other.fIncludePath;
bb885a9e 379 fCloseSE = other.fCloseSE;
0df6ccf2 380 fFriendChainName = other.fFriendChainName;
c6cb3634 381 fJobTag = other.fJobTag;
648174cf 382 fOutputSingle = other.fOutputSingle;
5fce53f4 383 fRunPrefix = other.fRunPrefix;
3bdcb562 384 fProofCluster = other.fProofCluster;
385 fProofDataSet = other.fProofDataSet;
386 fFileForTestMode = other.fFileForTestMode;
387 fRootVersionForProof = other.fRootVersionForProof;
388 fAliRootMode = other.fAliRootMode;
f47d5cb4 389 fMergeDirName = other.fMergeDirName;
c57f56b7 390 if (other.fInputFiles) {
391 fInputFiles = new TObjArray();
392 TIter next(other.fInputFiles);
393 TObject *obj;
394 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
395 fInputFiles->SetOwner();
396 }
4e5c5506 397 if (other.fPackages) {
398 fPackages = new TObjArray();
399 TIter next(other.fPackages);
400 TObject *obj;
401 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
402 fPackages->SetOwner();
403 }
77f34eae 404 if (other.fModules) {
405 fModules = new TObjArray();
406 fModules->SetOwner();
407 TIter next(other.fModules);
408 AliAnalysisTaskCfg *mod, *crt;
409 while ((crt=(AliAnalysisTaskCfg*)next())) {
410 mod = new AliAnalysisTaskCfg(*crt);
411 fModules->Add(mod);
412 }
413 }
c57f56b7 414 }
415 return *this;
416}
417
77f34eae 418//______________________________________________________________________________
419void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
420{
421// Adding a module. Checks if already existing. Becomes owned by this.
422 if (!module) return;
423 if (GetModule(module->GetName())) {
424 Error("AddModule", "A module having the same name %s already added", module->GetName());
425 return;
426 }
427 if (!fModules) {
428 fModules = new TObjArray();
429 fModules->SetOwner();
430 }
431 fModules->Add(module);
432}
433
434//______________________________________________________________________________
435void AliAnalysisAlien::AddModules(TObjArray *list)
436{
437// Adding a list of modules. Checks if already existing. Becomes owned by this.
438 TIter next(list);
439 AliAnalysisTaskCfg *module;
440 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
441}
442
443//______________________________________________________________________________
444Bool_t AliAnalysisAlien::CheckDependencies()
445{
446// Check if all dependencies are satisfied. Reorder modules if needed.
447 Int_t nmodules = GetNmodules();
448 if (!nmodules) {
449 Warning("CheckDependencies", "No modules added yet to check their dependencies");
450 return kTRUE;
451 }
452 AliAnalysisTaskCfg *mod = 0;
453 AliAnalysisTaskCfg *dep = 0;
454 TString depname;
455 Int_t i, j, k;
456 for (i=0; i<nmodules; i++) {
457 mod = (AliAnalysisTaskCfg*) fModules->At(i);
458 Int_t ndeps = mod->GetNdeps();
459 Int_t istart = i;
460 for (j=0; j<ndeps; j++) {
461 depname = mod->GetDependency(j);
462 dep = GetModule(depname);
463 if (!dep) {
464 Error("CheckDependencies","Dependency %s not added for module %s",
465 depname.Data(), mod->GetName());
466 return kFALSE;
467 }
468 if (dep->NeedsDependency(mod->GetName())) {
469 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
470 mod->GetName(), dep->GetName());
471 return kFALSE;
472 }
473 Int_t idep = fModules->IndexOf(dep);
474 // The dependency task must come first
475 if (idep>i) {
476 // Remove at idep and move all objects below up one slot
477 // down to index i included.
478 fModules->RemoveAt(idep);
3e40fd4c 479 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 480 fModules->AddAt(dep, i++);
481 }
482 //Redo from istart if dependencies were inserted
483 if (i>istart) i=istart-1;
484 }
485 }
486 return kTRUE;
487}
488
3e40fd4c 489//______________________________________________________________________________
490AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
491{
492// Create the analysis manager and optionally execute the macro in filename.
493 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
494 if (mgr) return mgr;
495 mgr = new AliAnalysisManager(name);
496 mgr->SetGridHandler((AliAnalysisGrid*)this);
497 if (strlen(filename)) {
498 TString line = gSystem->ExpandPathName(filename);
499 line.Prepend(".x ");
500 gROOT->ProcessLine(line.Data());
501 }
502 return mgr;
503}
504
77f34eae 505//______________________________________________________________________________
506Int_t AliAnalysisAlien::GetNmodules() const
507{
508// Get number of modules.
509 if (!fModules) return 0;
510 return fModules->GetEntries();
511}
512
513//______________________________________________________________________________
514AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
515{
516// Get a module by name.
517 if (!fModules) return 0;
518 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
519}
520
521//______________________________________________________________________________
522Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
523{
524// Load a given module.
525 if (mod->IsLoaded()) return kTRUE;
526 Int_t ndeps = mod->GetNdeps();
527 TString depname;
528 for (Int_t j=0; j<ndeps; j++) {
529 depname = mod->GetDependency(j);
530 AliAnalysisTaskCfg *dep = GetModule(depname);
531 if (!dep) {
532 Error("LoadModule","Dependency %s not existing for module %s",
533 depname.Data(), mod->GetName());
534 return kFALSE;
535 }
536 if (!LoadModule(dep)) {
537 Error("LoadModule","Dependency %s for module %s could not be loaded",
538 depname.Data(), mod->GetName());
539 return kFALSE;
540 }
541 }
542 // Load libraries for the module
543 if (!mod->CheckLoadLibraries()) {
544 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
545 return kFALSE;
546 }
547 // Execute the macro
548 if (mod->ExecuteMacro()<0) {
549 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
550 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
551 return kFALSE;
552 }
553 // Configure dependencies
554 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
555 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
556 mod->GetConfigMacro()->GetTitle(), mod->GetName());
557 return kFALSE;
558 }
559 return kTRUE;
560}
561
562//______________________________________________________________________________
21ca8e59 563Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 564{
565// Generate test macros for a single module or for the full train.
566 if (strlen(modname)) {
21ca8e59 567 if (!CheckDependencies()) return kFALSE;
568 AliAnalysisTaskCfg *mod = GetModule(modname);
569 if (!mod) {
570 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
571 return kFALSE;
572 }
573 if (!LoadModule(mod)) return kFALSE;
574 } else if (!LoadModules()) return kFALSE;
575 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
576 if (!mgr->InitAnalysis()) return kFALSE;
577 mgr->PrintStatus();
578 SetLocalTest(kTRUE);
579 Int_t productionMode = fProductionMode;
580 SetProductionMode();
581 TString macro = fAnalysisMacro;
582 TString executable = fExecutable;
583 TString validation = fValidationScript;
584 TString execCommand = fExecutableCommand;
585 SetAnalysisMacro(Form("%s.C", name));
586 SetExecutable(Form("%s.sh", name));
587 SetExecutableCommand("aliroot -b -q ");
588 SetValidationScript(Form("%s_validation.sh", name));
589 WriteAnalysisFile();
590 WriteAnalysisMacro();
591 WriteExecutable();
592 WriteValidationScript();
593 SetLocalTest(kFALSE);
594 SetProductionMode(productionMode);
595 fAnalysisMacro = macro;
596 fExecutable = executable;
597 fExecutableCommand = execCommand;
598 fValidationScript = validation;
77f34eae 599 return kTRUE;
600}
601
602//______________________________________________________________________________
603Bool_t AliAnalysisAlien::LoadModules()
604{
605// Load all modules by executing the AddTask macros. Checks first the dependencies.
21ca8e59 606 Int_t nmodules = GetNmodules();
607 if (!nmodules) {
608 Warning("LoadModules", "No module to be loaded");
609 return kTRUE;
610 }
3e40fd4c 611 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
612 if (!mgr) {
613 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
614 return kFALSE;
615 }
77f34eae 616 if (!CheckDependencies()) return kFALSE;
21ca8e59 617 nmodules = GetNmodules();
77f34eae 618 AliAnalysisTaskCfg *mod;
21ca8e59 619 for (Int_t imod=0; imod<nmodules; imod++) {
620 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 621 if (!LoadModule(mod)) return kFALSE;
622 }
623 // All modules are loaded. Adjust the library list.
77f34eae 624 fAdditionalLibs = "";
625 TString lib;
21ca8e59 626 for (Int_t imod=0; imod<nmodules; imod++) {
627 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 628 Int_t nlibs = mod->GetNlibs();
629 for (Int_t i=0; i<nlibs; i++) {
630 lib = mod->GetLibrary(i);
631 if (fAdditionalLibs.Contains(lib)) continue;
632 lib = Form("lib%s.so", lib.Data());
633 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
634 fAdditionalLibs += lib;
635 }
3e40fd4c 636 }
77f34eae 637 return kTRUE;
638}
639
74013535 640//______________________________________________________________________________
641void AliAnalysisAlien::SetRunPrefix(const char *prefix)
642{
643// Set the run number format. Can be a prefix or a format like "%09d"
644 fRunPrefix = prefix;
645 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
646}
647
f965131e 648//______________________________________________________________________________
649void AliAnalysisAlien::AddIncludePath(const char *path)
650{
651// Add include path in the remote analysis macro.
652 TString p(path);
653 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
654 else fIncludePath += Form("-I%s ", path);
655}
656
c57f56b7 657//______________________________________________________________________________
658void AliAnalysisAlien::AddRunNumber(Int_t run)
659{
660// Add a run number to the list of runs to be processed.
661 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 662 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 663}
664
98690895 665//______________________________________________________________________________
666void AliAnalysisAlien::AddRunList(const char* runList)
667{
668// Add several runs into the list of runs; they are expected to be separated by a blank character.
669 TString sList = runList;
670 TObjArray *list = sList.Tokenize(" ");
671 Int_t n = list->GetEntries();
672 for (Int_t i = 0; i < n; i++) {
673 TObjString *os = (TObjString*)list->At(i);
674 AddRunNumber(os->GetString().Atoi());
675 }
676 delete list;
677}
678
ee75cfc3 679//______________________________________________________________________________
680void AliAnalysisAlien::AddRunNumber(const char* run)
681{
682// Add a run number to the list of runs to be processed.
7aa5d539 683 TString runs = run;
684 TObjString *os;
685 TObjArray *arr = runs.Tokenize(" ");
686 TIter next(arr);
687 TString prefix;
688 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
689 while ((os=(TObjString*)next())){
690 if (fRunNumbers.Length()) fRunNumbers += " ";
691 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
692 }
693 delete arr;
ee75cfc3 694}
695
c57f56b7 696//______________________________________________________________________________
697void AliAnalysisAlien::AddDataFile(const char *lfn)
698{
699// Adds a data file to the input to be analysed. The file should be a valid LFN
700// or point to an existing file in the alien workdir.
701 if (!fInputFiles) fInputFiles = new TObjArray();
702 fInputFiles->Add(new TObjString(lfn));
703}
648174cf 704
705//______________________________________________________________________________
706void AliAnalysisAlien::AddExternalPackage(const char *package)
707{
708// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
709 if (fExternalPackages) fExternalPackages += " ";
710 fExternalPackages += package;
711}
712
c57f56b7 713//______________________________________________________________________________
714Bool_t AliAnalysisAlien::Connect()
715{
716// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
717 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 718 if (fProductionMode) return kTRUE;
c57f56b7 719 if (!gGrid) {
720 Info("Connect", "Trying to connect to AliEn ...");
721 TGrid::Connect("alien://");
722 }
723 if (!gGrid || !gGrid->IsConnected()) {
724 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
725 return kFALSE;
726 }
727 fUser = gGrid->GetUser();
728 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
729 return kTRUE;
730}
731
732//______________________________________________________________________________
733void AliAnalysisAlien::CdWork()
734{
735// Check validity of alien workspace. Create directory if possible.
736 if (!Connect()) {
737 Error("CdWork", "Alien connection required");
738 return;
739 }
740 TString homedir = gGrid->GetHomeDirectory();
741 TString workdir = homedir + fGridWorkingDir;
923e2ca5 742 if (DirectoryExists(workdir)) {
743 gGrid->Cd(workdir);
744 return;
745 }
746 // Work directory not existing - create it
747 gGrid->Cd(homedir);
b93f8109 748 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 749 gGrid->Cd(fGridWorkingDir);
d3b18c4c 750 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 751 } else {
d3b18c4c 752 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 753 workdir.Data(), homedir.Data());
754 fGridWorkingDir = "";
755 }
c57f56b7 756}
757
348be253 758//______________________________________________________________________________
759Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
760{
761// Check if file copying is possible.
d3b18c4c 762 if (fProductionMode) return kTRUE;
348be253 763 if (!Connect()) {
764 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
765 return kFALSE;
766 }
d3339be3 767 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
768 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 769 // Check if alien_CLOSE_SE is defined
770 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
771 if (!closeSE.IsNull()) {
772 Info("CheckFileCopy", "Your current close storage is pointing to: \
773 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
774 } else {
775 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
776 }
777 // Check if grid directory exists.
778 if (!DirectoryExists(alienpath)) {
779 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
780 return kFALSE;
781 }
d3339be3 782 TFile f("plugin_test_copy", "RECREATE");
348be253 783 // User may not have write permissions to current directory
784 if (f.IsZombie()) {
785 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
786 gSystem->WorkingDirectory());
787 return kFALSE;
788 }
789 f.Close();
d3339be3 790 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
791 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
792 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 793 \n# 1. Make sure you have write permissions there. If this is the case: \
794 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
795 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
796 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
797 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
798 gSystem->Unlink(f.GetName());
799 return kFALSE;
800 }
801 gSystem->Unlink(f.GetName());
d3339be3 802 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
803 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 804 return kTRUE;
805}
806
c57f56b7 807//______________________________________________________________________________
808Bool_t AliAnalysisAlien::CheckInputData()
809{
810// Check validity of input data. If necessary, create xml files.
d3b18c4c 811 if (fProductionMode) return kTRUE;
a8739e8a 812 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 813 if (!fGridDataDir.Length()) {
814 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
815 return kFALSE;
816 }
9f020952 817 if (fMergeViaJDL) {
818 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
819 return kFALSE;
820 }
d2a409b2 821 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 822 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
823 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 824 return kTRUE;
c57f56b7 825 }
826 // Process declared files
a2f5fc01 827 Bool_t isCollection = kFALSE;
828 Bool_t isXml = kFALSE;
829 Bool_t useTags = kFALSE;
c57f56b7 830 Bool_t checked = kFALSE;
d3b18c4c 831 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 832 TString file;
833 TString workdir = gGrid->GetHomeDirectory();
834 workdir += fGridWorkingDir;
835 if (fInputFiles) {
836 TObjString *objstr;
837 TIter next(fInputFiles);
838 while ((objstr=(TObjString*)next())) {
839 file = workdir;
840 file += "/";
841 file += objstr->GetString();
842 // Store full lfn path
843 if (FileExists(file)) objstr->SetString(file);
844 else {
845 file = objstr->GetName();
846 if (!FileExists(objstr->GetName())) {
847 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
848 objstr->GetName(), workdir.Data());
849 return kFALSE;
850 }
851 }
852 Bool_t iscoll, isxml, usetags;
853 CheckDataType(file, iscoll, isxml, usetags);
854 if (!checked) {
855 checked = kTRUE;
a2f5fc01 856 isCollection = iscoll;
857 isXml = isxml;
858 useTags = usetags;
859 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 860 } else {
a2f5fc01 861 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 862 Error("CheckInputData", "Some conflict was found in the types of inputs");
863 return kFALSE;
864 }
865 }
866 }
867 }
868 // Process requested run numbers
a8739e8a 869 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 870 // Check validity of alien data directory
871 if (!fGridDataDir.Length()) {
872 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
873 return kFALSE;
874 }
923e2ca5 875 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 876 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
877 return kFALSE;
878 }
a2f5fc01 879 if (isCollection) {
c57f56b7 880 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
881 return kFALSE;
882 }
883
a2f5fc01 884 if (checked && !isXml) {
c57f56b7 885 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
886 return kFALSE;
887 }
888 // Check validity of run number(s)
889 TObjArray *arr;
890 TObjString *os;
74013535 891 TString format;
319593fb 892 Int_t nruns = 0;
904f9f5f 893 TString schunk, schunk2;
c57f56b7 894 TString path;
895 if (!checked) {
896 checked = kTRUE;
a2f5fc01 897 useTags = fDataPattern.Contains("tag");
898 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 899 }
a2f5fc01 900 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 901 Error("CheckInputData", "Cannot mix input files using/not using tags");
902 return kFALSE;
903 }
904 if (fRunNumbers.Length()) {
a8739e8a 905 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 906 arr = fRunNumbers.Tokenize(" ");
907 TIter next(arr);
908 while ((os=(TObjString*)next())) {
909 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 910 if (!DirectoryExists(path)) {
911 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 912 continue;
c57f56b7 913 }
914 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
915 TString msg = "\n##### file: ";
916 msg += path;
917 msg += " type: xml_collection;";
a2f5fc01 918 if (useTags) msg += " using_tags: Yes";
c57f56b7 919 else msg += " using_tags: No";
23329835 920 Info("CheckDataType", "%s", msg.Data());
319593fb 921 if (fNrunsPerMaster<2) {
d2a409b2 922 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 923 } else {
924 nruns++;
925 if (((nruns-1)%fNrunsPerMaster) == 0) {
926 schunk = os->GetString();
927 }
928 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
929 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 930 AddDataFile(schunk);
319593fb 931 }
c57f56b7 932 }
933 delete arr;
a8739e8a 934 } else {
935 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
936 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 937 format = Form("%%s/%s ", fRunPrefix.Data());
938 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 939 if (!DirectoryExists(path)) {
a8739e8a 940 continue;
941 }
74013535 942 format = Form("%%s/%s.xml", fRunPrefix.Data());
943 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 944 TString msg = "\n##### file: ";
945 msg += path;
946 msg += " type: xml_collection;";
a2f5fc01 947 if (useTags) msg += " using_tags: Yes";
a8739e8a 948 else msg += " using_tags: No";
23329835 949 Info("CheckDataType", "%s", msg.Data());
319593fb 950 if (fNrunsPerMaster<2) {
74013535 951 format = Form("%s.xml", fRunPrefix.Data());
952 AddDataFile(Form(format.Data(),irun));
319593fb 953 } else {
954 nruns++;
955 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 956 schunk = Form(fRunPrefix.Data(),irun);
319593fb 957 }
74013535 958 format = Form("_%s.xml", fRunPrefix.Data());
959 schunk2 = Form(format.Data(), irun);
319593fb 960 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 961 schunk += schunk2;
d2a409b2 962 AddDataFile(schunk);
319593fb 963 }
a8739e8a 964 }
904f9f5f 965 if (!fInputFiles) {
966 schunk += schunk2;
967 AddDataFile(schunk);
968 }
c57f56b7 969 }
970 return kTRUE;
971}
972
21ca8e59 973//______________________________________________________________________________
974Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *anchorfile)
975{
976// Copy data from the given grid directory according a pattern and make a local
977// dataset.
978 if (!Connect()) {
979 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
980 return kFALSE;
981 }
982 if (!DirectoryExists(griddir)) {
983 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
984 return kFALSE;
985 }
986 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
987 printf("Running command: %s\n", command.Data());
988 TGridResult *res = gGrid->Command(command);
989 Int_t nfound = res->GetEntries();
990 if (!nfound) {
991 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
992 return kFALSE;
993 }
994 printf("... found %d files. Copying locally ...\n", nfound);
995 // Copy files locally
996 ofstream out;
997 out.open(output, ios::out);
998 TMap *map;
999 TString turl, dirname, filename, temp;
1000 TString cdir = gSystem->WorkingDirectory();
1001 gSystem->MakeDirectory("data");
1002 gSystem->ChangeDirectory("data");
1003 for (Int_t i=0; i<nfound; i++) {
1004 map = (TMap*)res->At(i);
1005 turl = map->GetValue("turl")->GetName();
1006 filename = gSystem->BaseName(turl.Data());
1007 dirname = gSystem->DirName(turl.Data());
1008 dirname = gSystem->BaseName(dirname.Data());
1009 gSystem->MakeDirectory(dirname);
1010 if (TFile::Cp(turl, Form("file:./%s/%s", dirname.Data(), filename.Data()))) {
1011 if (strlen(anchorfile)) filename = Form("%s#%s", filename.Data(), anchorfile);
1012 out << cdir << "/data/" << Form("%s/%s", dirname.Data(), filename.Data()) << endl;
1013 }
1014 }
1015 gSystem->ChangeDirectory(cdir);
1016 delete res;
1017 return kTRUE;
1018}
1019
c57f56b7 1020//______________________________________________________________________________
1021Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1022{
1023// Create dataset for the grid data directory + run number.
3af861ff 1024 const Int_t gMaxEntries = 15000;
d3b18c4c 1025 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1026 if (!Connect()) {
1027 Error("CreateDataset", "Cannot create dataset with no grid connection");
1028 return kFALSE;
1029 }
1030
1031 // Cd workspace
d3b18c4c 1032 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1033 TString workdir = gGrid->GetHomeDirectory();
1034 workdir += fGridWorkingDir;
1035
1036 // Compose the 'find' command arguments
74013535 1037 TString format;
c57f56b7 1038 TString command;
1039 TString options = "-x collection ";
bb885a9e 1040 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1041 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1042 TString conditions = "";
3af861ff 1043 Int_t nstart = 0;
1044 Int_t ncount = 0;
1045 Int_t stage = 0;
c57f56b7 1046 TString file;
1047 TString path;
319593fb 1048 Int_t nruns = 0;
904f9f5f 1049 TString schunk, schunk2;
ab254fd1 1050 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1051 if (!fRunNumbers.Length() && !fRunRange[0]) {
1052 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1053 // Make a single data collection from data directory.
1054 path = fGridDataDir;
923e2ca5 1055 if (!DirectoryExists(path)) {
d2a409b2 1056 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1057 return kFALSE;
1058 }
923e2ca5 1059// CdWork();
d2a409b2 1060 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1061 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1062 while (1) {
1063 ncount = 0;
1064 stage++;
1065 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1066 command = "find ";
1067 command += Form("%s -o %d ",options.Data(), nstart);
1068 command += path;
1069 command += " ";
1070 command += pattern;
1071 command += conditions;
1072 printf("command: %s\n", command.Data());
1073 TGridResult *res = gGrid->Command(command);
1074 if (res) delete res;
1075 // Write standard output to file
1076 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1077 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1078 Bool_t nullFile = kFALSE;
1079 if (!hasGrep) {
1080 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1081 } else {
1082 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1083 if (nullFile) {
1084 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1085 gSystem->Exec("rm -f __tmp*");
1086 return kFALSE;
1087 }
1088 TString line;
1089 ifstream in;
1090 in.open("__tmp__");
1091 in >> line;
1092 in.close();
1093 gSystem->Exec("rm -f __tmp__");
1094 ncount = line.Atoi();
1095 }
1096 }
1097 if (ncount == gMaxEntries) {
1098 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1099 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1100 if (!cbase) cbase = cadd;
1101 else {
1102 cbase->Add(cadd);
1103 delete cadd;
defd7a3a 1104 }
3af861ff 1105 nstart += ncount;
1106 } else {
1107 if (cbase) {
1108 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1109 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1110 cbase->Add(cadd);
1111 delete cadd;
1112 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1113 delete cbase; cbase = 0;
1114 } else {
1115 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1116 }
1117 gSystem->Exec("rm -f __tmp*");
1118 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1119 break;
1120 }
0f389141 1121 }
1122 Bool_t fileExists = FileExists(file);
1123 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1124 // Copy xml file to alien space
0f389141 1125 if (fileExists) gGrid->Rm(file);
d2a409b2 1126 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1127 if (!FileExists(file)) {
1128 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1129 return kFALSE;
1130 }
1131 // Update list of files to be processed.
1132 }
1133 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1134 return kTRUE;
1135 }
c57f56b7 1136 // Several runs
a2f5fc01 1137 Bool_t nullResult = kTRUE;
a8739e8a 1138 if (fRunNumbers.Length()) {
1139 TObjArray *arr = fRunNumbers.Tokenize(" ");
1140 TObjString *os;
1141 TIter next(arr);
1142 while ((os=(TObjString*)next())) {
3af861ff 1143 nstart = 0;
1144 stage = 0;
dc030a24 1145 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1146 if (!DirectoryExists(path)) continue;
1147// CdWork();
a8739e8a 1148 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1149 else file = Form("%s.xml", os->GetString().Data());
319593fb 1150 // If local collection file does not exist, create it via 'find' command.
3af861ff 1151 while (1) {
1152 ncount = 0;
1153 stage++;
1154 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1155 command = "find ";
1156 command += Form("%s -o %d ",options.Data(), nstart);
1157 command += path;
1158 command += pattern;
1159 command += conditions;
1160 TGridResult *res = gGrid->Command(command);
1161 if (res) delete res;
1162 // Write standard output to file
1163 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1164 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1165 Bool_t nullFile = kFALSE;
1166 if (!hasGrep) {
1167 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1168 } else {
1169 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1170 if (nullFile) {
1171 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1172 gSystem->Exec("rm -f __tmp*");
1173 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1174 break;
3af861ff 1175 }
1176 TString line;
1177 ifstream in;
1178 in.open("__tmp__");
1179 in >> line;
1180 in.close();
1181 gSystem->Exec("rm -f __tmp__");
1182 ncount = line.Atoi();
1183 }
1184 nullResult = kFALSE;
1185 }
1186 if (ncount == gMaxEntries) {
1187 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1188 if (fNrunsPerMaster > 1) {
1189 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1190 file.Data(),gMaxEntries);
1191 return kFALSE;
1192 }
1193 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1194 if (!cbase) cbase = cadd;
1195 else {
1196 cbase->Add(cadd);
1197 delete cadd;
defd7a3a 1198 }
3af861ff 1199 nstart += ncount;
1200 } else {
c89a84bc 1201 if (cbase && fNrunsPerMaster<2) {
3af861ff 1202 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1203 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1204 cbase->Add(cadd);
1205 delete cadd;
1206 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1207 delete cbase; cbase = 0;
1208 } else {
1209 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1210 }
1211 gSystem->Exec("rm -f __tmp*");
1212 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1213 break;
d3339be3 1214 }
3af861ff 1215 }
a8739e8a 1216 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1217 // Check if there is one run per master job.
1218 if (fNrunsPerMaster<2) {
1219 if (FileExists(file)) {
0f389141 1220 if (fOverwriteMode) gGrid->Rm(file);
1221 else {
1222 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1223 continue;
1224 }
319593fb 1225 }
1226 // Copy xml file to alien space
1227 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1228 if (!FileExists(file)) {
1229 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1230 delete arr;
1231 return kFALSE;
1232 }
1233 } else {
1234 nruns++;
1235 if (((nruns-1)%fNrunsPerMaster) == 0) {
1236 schunk = os->GetString();
1237 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1238 } else {
1239 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1240 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1241 cbase->Add(cadd);
1242 delete cadd;
1243 }
1244 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1245 continue;
1246 }
1247 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1248 if (FileExists(schunk)) {
1249 if (fOverwriteMode) gGrid->Rm(file);
1250 else {
1251 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1252 continue;
1253 }
319593fb 1254 }
84fcd93f 1255 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1256 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1257 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1258 if (!FileExists(schunk)) {
1259 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1260 delete arr;
1261 return kFALSE;
1262 }
d3339be3 1263 }
a8739e8a 1264 }
1265 delete arr;
a2f5fc01 1266 if (nullResult) {
d3339be3 1267 Error("CreateDataset", "No valid dataset corresponding to the query!");
1268 return kFALSE;
1269 }
a8739e8a 1270 } else {
1271 // Process a full run range.
1272 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1273 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1274 nstart = 0;
1275 stage = 0;
74013535 1276 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1277 if (!DirectoryExists(path)) continue;
1278// CdWork();
74013535 1279 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1280 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1281 else file = Form(format.Data(), irun);
0f389141 1282 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1283 if (fOverwriteMode) gGrid->Rm(file);
1284 else {
1285 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1286 continue;
1287 }
a8739e8a 1288 }
319593fb 1289 // If local collection file does not exist, create it via 'find' command.
3af861ff 1290 while (1) {
1291 ncount = 0;
1292 stage++;
1293 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1294 command = "find ";
1295 command += Form("%s -o %d ",options.Data(), nstart);
1296 command += path;
1297 command += pattern;
1298 command += conditions;
1299 TGridResult *res = gGrid->Command(command);
1300 if (res) delete res;
1301 // Write standard output to file
1302 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1303 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1304 Bool_t nullFile = kFALSE;
1305 if (!hasGrep) {
1306 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1307 } else {
1308 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1309 if (nullFile) {
1310 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1311 gSystem->Exec("rm -f __tmp*");
1604c9ed 1312 break;
3af861ff 1313 }
1314 TString line;
1315 ifstream in;
1316 in.open("__tmp__");
1317 in >> line;
1318 in.close();
1319 gSystem->Exec("rm -f __tmp__");
1320 ncount = line.Atoi();
1321 }
1322 nullResult = kFALSE;
1323 }
1324 if (ncount == gMaxEntries) {
1325 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1326 if (fNrunsPerMaster > 1) {
1327 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1328 file.Data(),gMaxEntries);
1329 return kFALSE;
1330 }
1331 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1332 if (!cbase) cbase = cadd;
1333 else {
1334 cbase->Add(cadd);
1335 delete cadd;
defd7a3a 1336 }
3af861ff 1337 nstart += ncount;
1338 } else {
c89a84bc 1339 if (cbase && fNrunsPerMaster<2) {
3af861ff 1340 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1341 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1342 cbase->Add(cadd);
1343 delete cadd;
1344 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1345 delete cbase; cbase = 0;
1346 } else {
1347 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1348 }
1349 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1350 break;
d3339be3 1351 }
319593fb 1352 }
a8739e8a 1353 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1354 // Check if there is one run per master job.
1355 if (fNrunsPerMaster<2) {
1356 if (FileExists(file)) {
0f389141 1357 if (fOverwriteMode) gGrid->Rm(file);
1358 else {
1359 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1360 continue;
1361 }
319593fb 1362 }
1363 // Copy xml file to alien space
1364 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1365 if (!FileExists(file)) {
1366 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1367 return kFALSE;
1368 }
1369 } else {
1370 nruns++;
95e5b448 1371 // Check if the collection for the chunk exist locally.
1372 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1373 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1374 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1375 else continue;
1376 }
84fcd93f 1377 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1378 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1379 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1380 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1381 } else {
1382 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1383 cbase->Add(cadd);
1384 delete cadd;
1385 }
74013535 1386 format = Form("%%s_%s.xml", fRunPrefix.Data());
1387 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1388 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1389 continue;
1390 }
904f9f5f 1391 schunk = schunk2;
319593fb 1392 if (FileExists(schunk)) {
0f389141 1393 if (fOverwriteMode) gGrid->Rm(schunk);
1394 else {
1395 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1396 continue;
1397 }
319593fb 1398 }
84fcd93f 1399 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1400 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1401 if (FileExists(schunk)) {
0f389141 1402 if (fOverwriteMode) gGrid->Rm(schunk);
1403 else {
1404 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1405 continue;
1406 }
95e5b448 1407 }
319593fb 1408 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1409 if (!FileExists(schunk)) {
1410 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1411 return kFALSE;
1412 }
1413 }
c57f56b7 1414 }
a2f5fc01 1415 if (nullResult) {
d3339be3 1416 Error("CreateDataset", "No valid dataset corresponding to the query!");
1417 return kFALSE;
1418 }
a8739e8a 1419 }
c57f56b7 1420 return kTRUE;
1421}
1422
1423//______________________________________________________________________________
1424Bool_t AliAnalysisAlien::CreateJDL()
1425{
1426// Generate a JDL file according to current settings. The name of the file is
1427// specified by fJDLName.
1428 Bool_t error = kFALSE;
1429 TObjArray *arr = 0;
1430 Bool_t copy = kTRUE;
d3b18c4c 1431 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1432 Bool_t generate = kTRUE;
1433 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1434 if (!Connect()) {
1435 Error("CreateJDL", "Alien connection required");
1436 return kFALSE;
1437 }
1438 // Check validity of alien workspace
d3b18c4c 1439 TString workdir;
1440 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1441 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1442 workdir += fGridWorkingDir;
1443 if (generate) {
1444 TObjString *os;
1445 if (!fInputFiles) {
1446 Error("CreateJDL()", "Define some input files for your analysis.");
1447 error = kTRUE;
1448 }
1449 // Compose list of input files
1450 // Check if output files were defined
1451 if (!fOutputFiles.Length()) {
1452 Error("CreateJDL", "You must define at least one output file");
1453 error = kTRUE;
1454 }
1455 // Check if an output directory was defined and valid
1456 if (!fGridOutputDir.Length()) {
1457 Error("CreateJDL", "You must define AliEn output directory");
1458 error = kTRUE;
1459 } else {
d3b18c4c 1460 if (!fProductionMode) {
1461 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1462 if (!DirectoryExists(fGridOutputDir)) {
1463 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1464 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1465 } else {
1466 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1467 // error = kTRUE;
1468 }
67a3b254 1469 } else {
1470 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1471 }
1472 gGrid->Cd(workdir);
1473 }
c57f56b7 1474 }
1475 // Exit if any error up to now
1476 if (error) return kFALSE;
1477 // Set JDL fields
0f389141 1478 if (!fUser.IsNull()) {
1479 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1480 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1481 }
1482 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1483 TString mergeExec = fExecutable;
1484 mergeExec.ReplaceAll(".sh", "_merge.sh");
1485 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1486 mergeExec.ReplaceAll(".sh", ".C");
1487 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1488 if (!fArguments.IsNull())
1489 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1490 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1491 else {
1492 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1493 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1494 }
b3e07543 1495
1f0d1ca2 1496 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1497 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1498 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1499 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1500
0f389141 1501 if (fMaxInitFailed > 0) {
c57f56b7 1502 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1503 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1504 }
1505 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1506 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1507 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1508 }
1509 if (!IsOneStageMerging()) {
f47d5cb4 1510 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1511 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1512 }
1513 if (fSplitMode.Length()) {
c57f56b7 1514 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1515 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1516 }
1517 fMergingJDL->SetValue("Split", "\"se\"");
1518 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1519 if (!fAliROOTVersion.IsNull()) {
1520 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1521 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1522 }
1523 if (!fROOTVersion.IsNull()) {
c57f56b7 1524 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1525 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1526 }
1527 if (!fAPIVersion.IsNull()) {
c57f56b7 1528 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1529 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1530 }
648174cf 1531 if (!fExternalPackages.IsNull()) {
1532 arr = fExternalPackages.Tokenize(" ");
1533 TIter next(arr);
1534 while ((os=(TObjString*)next())) {
1535 TString pkgname = os->GetString();
1536 Int_t index = pkgname.Index("::");
1537 TString pkgversion = pkgname(index+2, pkgname.Length());
1538 pkgname.Remove(index);
1539 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1540 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1541 }
1542 delete arr;
1543 }
0f389141 1544 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1545 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1546 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1547 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1548 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1549 TString analysisFile = fExecutable;
1550 analysisFile.ReplaceAll(".sh", ".root");
1551 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1552 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1553 if (fAdditionalLibs.Length()) {
1554 arr = fAdditionalLibs.Tokenize(" ");
1555 TIter next(arr);
1556 while ((os=(TObjString*)next())) {
1557 if (os->GetString().Contains(".so")) continue;
1558 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1559 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1560 }
1561 delete arr;
1562 }
4e5c5506 1563 if (fPackages) {
1564 TIter next(fPackages);
1565 TObject *obj;
0f389141 1566 while ((obj=next())) {
4e5c5506 1567 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1568 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1569 }
4e5c5506 1570 }
c57f56b7 1571 if (fOutputArchive.Length()) {
1572 arr = fOutputArchive.Tokenize(" ");
1573 TIter next(arr);
0f389141 1574 Bool_t first = kTRUE;
1575 const char *comment = "Files to be archived";
1576 const char *comment1 = comment;
1577 while ((os=(TObjString*)next())) {
1578 if (!first) comment = NULL;
1579 if (!os->GetString().Contains("@") && fCloseSE.Length())
1580 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1581 else
1582 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1583 first = kFALSE;
1584 }
c57f56b7 1585 delete arr;
f790bc1b 1586 // Output archive for the merging jdl
1587 TString outputArchive;
1588 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1589 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1590 // Add normal output files, extra files + terminate files
1591 TString files = GetListOfFiles("outextter");
1592 // Do not register merge excludes
1593 if (!fMergeExcludes.IsNull()) {
1594 arr = fMergeExcludes.Tokenize(" ");
1595 TIter next1(arr);
1596 while ((os=(TObjString*)next1())) {
1597 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1598 files.ReplaceAll(os->GetString(),"");
1599 }
66d71516 1600 delete arr;
0f389141 1601 }
f790bc1b 1602 files.ReplaceAll(".root", "*.root");
b83f84f0 1603 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1604 } else {
95f84779 1605 TString files = fOutputArchive;
1606 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1607 outputArchive = files;
f790bc1b 1608 }
0f389141 1609 arr = outputArchive.Tokenize(" ");
705adb3e 1610 TIter next2(arr);
0f389141 1611 comment = comment1;
1612 first = kTRUE;
705adb3e 1613 while ((os=(TObjString*)next2())) {
0f389141 1614 if (!first) comment = NULL;
0cdf65a8 1615 TString currentfile = os->GetString();
0cdf65a8 1616 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1617 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1618 else
0cdf65a8 1619 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1620 first = kFALSE;
1621 }
1622 delete arr;
c57f56b7 1623 }
149d288c 1624 arr = fOutputFiles.Tokenize(",");
c57f56b7 1625 TIter next(arr);
0f389141 1626 Bool_t first = kTRUE;
f790bc1b 1627 const char *comment = "Files to be saved";
43da816a 1628 while ((os=(TObjString*)next())) {
e1eaf596 1629 // Ignore ouputs in jdl that are also in outputarchive
1630 TString sout = os->GetString();
f790bc1b 1631 sout.ReplaceAll("*", "");
1632 sout.ReplaceAll(".root", "");
e1eaf596 1633 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1634 if (fOutputArchive.Contains(sout)) continue;
0f389141 1635 if (!first) comment = NULL;
43da816a 1636 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1637 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1638 else
0f389141 1639 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1640 first = kFALSE;
f790bc1b 1641 if (fMergeExcludes.Contains(sout)) continue;
1642 if (!os->GetString().Contains("@") && fCloseSE.Length())
1643 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1644 else
1645 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1646 }
c57f56b7 1647 delete arr;
0f389141 1648 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1649 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1650 TString validationScript = fValidationScript;
0f389141 1651 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1652 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1653 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1654 if (fMasterResubmitThreshold) {
1655 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1656 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1657 }
d2a409b2 1658 // Write a jdl with 2 input parameters: collection name and output dir name.
1659 WriteJDL(copy);
c57f56b7 1660 }
1661 // Copy jdl to grid workspace
a8739e8a 1662 if (copy) {
b5e4aaa7 1663 // Check if an output directory was defined and valid
1664 if (!fGridOutputDir.Length()) {
1665 Error("CreateJDL", "You must define AliEn output directory");
1666 return kFALSE;
1667 } else {
1668 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1669 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1670 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1671 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1672 } else {
1673 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1674 return kFALSE;
1675 }
1676 }
1677 gGrid->Cd(workdir);
1678 }
648174cf 1679 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1680 TString mergeJDLName = fExecutable;
1681 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1682 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1683 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1684 if (fProductionMode) {
648174cf 1685 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1686 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1687 }
648174cf 1688 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1689 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1690 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1691 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1692 Fatal("","Terminating");
1693// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1694 if (fMergeViaJDL) {
1695 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1696// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1697 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1698 Fatal("","Terminating");
0f389141 1699 }
648174cf 1700 }
c57f56b7 1701 if (fAdditionalLibs.Length()) {
e7c71df0 1702 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1703 TObjString *os;
1704 TIter next(arr);
1705 while ((os=(TObjString*)next())) {
c57f56b7 1706 if (os->GetString().Contains(".so")) continue;
4e5c5506 1707 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1708 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1709// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1710 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1711 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1712 Fatal("","Terminating");
c57f56b7 1713 }
1714 delete arr;
1715 }
4e5c5506 1716 if (fPackages) {
1717 TIter next(fPackages);
1718 TObject *obj;
1719 while ((obj=next())) {
fdbbc7be 1720 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1721 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1722// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1723 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1724 Form("%s/%s", workdir.Data(), obj->GetName())))
1725 Fatal("","Terminating");
4e5c5506 1726 }
1727 }
c57f56b7 1728 }
1729 return kTRUE;
1730}
1731
a8739e8a 1732//______________________________________________________________________________
d2a409b2 1733Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1734{
1735// Writes one or more JDL's corresponding to findex. If findex is negative,
1736// all run numbers are considered in one go (jdl). For non-negative indices
1737// they correspond to the indices in the array fInputFiles.
1738 if (!fInputFiles) return kFALSE;
d3b18c4c 1739 TObject *os;
1740 TString workdir;
1741 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1742 workdir += fGridWorkingDir;
f47d5cb4 1743 TString stageName = "$2";
1744 if (fProductionMode) stageName = "$4";
1745 if (!fMergeDirName.IsNull()) {
1746 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1747 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1748 } else {
1749 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1750 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1751 }
d3b18c4c 1752 if (fProductionMode) {
a8739e8a 1753 TIter next(fInputFiles);
b3e07543 1754 while ((os=next())) {
d3b18c4c 1755 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1756 }
56754bf6 1757 if (!fOutputToRunNo)
1758 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1759 else
1760 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1761 } else {
1762 if (!fRunNumbers.Length() && !fRunRange[0]) {
1763 // One jdl with no parameters in case input data is specified by name.
1764 TIter next(fInputFiles);
1765 while ((os=next()))
1766 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1767 if (!fOutputSingle.IsNull())
1768 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1769 else {
1770 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1771 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1772 }
1773 } else {
1774 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1775 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1776 if (!fOutputSingle.IsNull()) {
1777 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1778 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1779 } else {
1780 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1781 }
1782 }
a8739e8a 1783 }
1784
a8739e8a 1785 // Generate the JDL as a string
1786 TString sjdl = fGridJDL->Generate();
0f389141 1787 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1788 // Final merge jdl
1789 if (!fMergeDirName.IsNull()) {
1790 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1791 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1792 } else {
1793 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1794 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1795 }
b3e07543 1796 TString sjdl2 = fMergingJDL->Generate();
1797 Int_t index, index1;
a8739e8a 1798 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1799 sjdl.ReplaceAll("(member", "\n (member");
1800 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1801 sjdl.ReplaceAll("{", "{\n ");
1802 sjdl.ReplaceAll("};", "\n};");
1803 sjdl.ReplaceAll("{\n \n", "{\n");
1804 sjdl.ReplaceAll("\n\n", "\n");
1805 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1806 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1807 sjdl1.ReplaceAll("(member", "\n (member");
1808 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1809 sjdl1.ReplaceAll("{", "{\n ");
1810 sjdl1.ReplaceAll("};", "\n};");
1811 sjdl1.ReplaceAll("{\n \n", "{\n");
1812 sjdl1.ReplaceAll("\n\n", "\n");
1813 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1814 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1815 sjdl2.ReplaceAll("(member", "\n (member");
1816 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1817 sjdl2.ReplaceAll("{", "{\n ");
1818 sjdl2.ReplaceAll("};", "\n};");
1819 sjdl2.ReplaceAll("{\n \n", "{\n");
1820 sjdl2.ReplaceAll("\n\n", "\n");
1821 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1822 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1823 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1824 index = sjdl.Index("JDLVariables");
1825 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1826 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1827 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1828 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1829 index = fJobTag.Index(":");
1830 if (index < 0) index = fJobTag.Length();
1831 TString jobTag = fJobTag;
f47d5cb4 1832 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1833 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1834 if (fProductionMode) {
1835 sjdl1.Prepend("# Generated merging jdl (production mode) \
1836 \n# $1 = full alien path to output directory to be merged \
1837 \n# $2 = train number \
1838 \n# $3 = production (like LHC10b) \
1839 \n# $4 = merging stage \
1840 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1841 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1842 sjdl2.Prepend("# Generated merging jdl \
1843 \n# $1 = full alien path to output directory to be merged \
1844 \n# $2 = train number \
1845 \n# $3 = production (like LHC10b) \
1846 \n# $4 = merging stage \
1847 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1848 } else {
1849 sjdl1.Prepend("# Generated merging jdl \
1850 \n# $1 = full alien path to output directory to be merged \
1851 \n# $2 = merging stage \
03335324 1852 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1853 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1854 sjdl2.Prepend("# Generated merging jdl \
1855 \n# $1 = full alien path to output directory to be merged \
1856 \n# $2 = merging stage \
03335324 1857 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1858 }
0f389141 1859 index = sjdl1.Index("JDLVariables");
1860 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1861 index = sjdl2.Index("JDLVariables");
1862 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1863 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1864 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1865 index = sjdl2.Index("Split =");
1866 if (index>=0) {
1867 index1 = sjdl2.Index("\n", index);
1868 sjdl2.Remove(index, index1-index+1);
1869 }
1870 index = sjdl2.Index("SplitMaxInputFileNumber");
1871 if (index>=0) {
1872 index1 = sjdl2.Index("\n", index);
1873 sjdl2.Remove(index, index1-index+1);
1874 }
1875 index = sjdl2.Index("InputDataCollection");
1876 if (index>=0) {
1877 index1 = sjdl2.Index(";", index);
1878 sjdl2.Remove(index, index1-index+1);
1879 }
1880 index = sjdl2.Index("InputDataListFormat");
1881 if (index>=0) {
1882 index1 = sjdl2.Index("\n", index);
1883 sjdl2.Remove(index, index1-index+1);
1884 }
1885 index = sjdl2.Index("InputDataList");
1886 if (index>=0) {
1887 index1 = sjdl2.Index("\n", index);
1888 sjdl2.Remove(index, index1-index+1);
1889 }
f47d5cb4 1890 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1891 // Write jdl to file
a8739e8a 1892 ofstream out;
d2a409b2 1893 out.open(fJDLName.Data(), ios::out);
a8739e8a 1894 if (out.bad()) {
d3b18c4c 1895 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1896 return kFALSE;
1897 }
1898 out << sjdl << endl;
b3e07543 1899 out.close();
0f389141 1900 TString mergeJDLName = fExecutable;
1901 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1902 if (fMergeViaJDL) {
1903 ofstream out1;
1904 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1905 if (out1.bad()) {
d3b18c4c 1906 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1907 return kFALSE;
1908 }
1909 out1 << sjdl1 << endl;
b3e07543 1910 out1.close();
1911 ofstream out2;
1912 TString finalJDL = mergeJDLName;
1913 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1914 out2.open(finalJDL.Data(), ios::out);
1915 if (out2.bad()) {
1916 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1917 return kFALSE;
1918 }
1919 out2 << sjdl2 << endl;
1920 out2.close();
0f389141 1921 }
a8739e8a 1922
1923 // Copy jdl to grid workspace
1924 if (!copy) {
d3b18c4c 1925 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1926 } else {
d2a409b2 1927 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1928 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1929 TString finalJDL = mergeJDLName;
1930 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1931 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1932 if (fProductionMode) {
b5fe9cba 1933 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1934 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1935 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1936 }
d2a409b2 1937 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1938 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1939 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1940 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1941// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1942 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
1943 Fatal("","Terminating");
0f389141 1944 if (fMergeViaJDL) {
b3e07543 1945 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1946// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1947// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
1948 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
1949 Fatal("","Terminating");
1950 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
1951 Fatal("","Terminating");
0f389141 1952 }
a8739e8a 1953 }
1954 return kTRUE;
1955}
1956
c57f56b7 1957//______________________________________________________________________________
5513444a 1958Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1959{
1960// Returns true if file exists.
5513444a 1961 if (!gGrid) return kFALSE;
2ea55496 1962 TString slfn = lfn;
1963 slfn.ReplaceAll("alien://","");
1964 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1965 if (!res) return kFALSE;
1966 TMap *map = dynamic_cast<TMap*>(res->At(0));
1967 if (!map) {
1968 delete res;
1969 return kFALSE;
1970 }
1971 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1972 if (!objs || !objs->GetString().Length()) {
1973 delete res;
1974 return kFALSE;
1975 }
1976 delete res;
1977 return kTRUE;
1978}
1979
923e2ca5 1980//______________________________________________________________________________
1981Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1982{
1983// Returns true if directory exists. Can be also a path.
1984 if (!gGrid) return kFALSE;
1985 // Check if dirname is a path
1986 TString dirstripped = dirname;
1987 dirstripped = dirstripped.Strip();
1988 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1989 TString dir = gSystem->BaseName(dirstripped);
1990 dir += "/";
1991 TString path = gSystem->DirName(dirstripped);
1992 TGridResult *res = gGrid->Ls(path, "-F");
1993 if (!res) return kFALSE;
1994 TIter next(res);
1995 TMap *map;
1996 TObject *obj;
1997 while ((map=dynamic_cast<TMap*>(next()))) {
1998 obj = map->GetValue("name");
1999 if (!obj) break;
2000 if (dir == obj->GetName()) {
2001 delete res;
2002 return kTRUE;
2003 }
2004 }
2005 delete res;
2006 return kFALSE;
2007}
2008
c57f56b7 2009//______________________________________________________________________________
a2f5fc01 2010void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2011{
2012// Check input data type.
a2f5fc01 2013 isCollection = kFALSE;
2014 isXml = kFALSE;
2015 useTags = kFALSE;
c57f56b7 2016 if (!gGrid) {
2017 Error("CheckDataType", "No connection to grid");
2018 return;
2019 }
a2f5fc01 2020 isCollection = IsCollection(lfn);
c57f56b7 2021 TString msg = "\n##### file: ";
2022 msg += lfn;
a2f5fc01 2023 if (isCollection) {
c57f56b7 2024 msg += " type: raw_collection;";
2025 // special treatment for collections
a2f5fc01 2026 isXml = kFALSE;
c57f56b7 2027 // check for tag files in the collection
2028 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2029 if (!res) {
2030 msg += " using_tags: No (unknown)";
23329835 2031 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2032 return;
2033 }
2034 const char* typeStr = res->GetKey(0, "origLFN");
2035 if (!typeStr || !strlen(typeStr)) {
2036 msg += " using_tags: No (unknown)";
23329835 2037 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2038 return;
2039 }
2040 TString file = typeStr;
a2f5fc01 2041 useTags = file.Contains(".tag");
2042 if (useTags) msg += " using_tags: Yes";
c57f56b7 2043 else msg += " using_tags: No";
23329835 2044 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2045 return;
2046 }
2047 TString slfn(lfn);
2048 slfn.ToLower();
a2f5fc01 2049 isXml = slfn.Contains(".xml");
2050 if (isXml) {
c57f56b7 2051 // Open xml collection and check if there are tag files inside
2052 msg += " type: xml_collection;";
2053 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2054 if (!coll) {
2055 msg += " using_tags: No (unknown)";
23329835 2056 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2057 return;
2058 }
2059 TMap *map = coll->Next();
2060 if (!map) {
2061 msg += " using_tags: No (unknown)";
23329835 2062 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2063 return;
2064 }
2065 map = (TMap*)map->GetValue("");
2066 TString file;
2067 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2068 useTags = file.Contains(".tag");
c57f56b7 2069 delete coll;
a2f5fc01 2070 if (useTags) msg += " using_tags: Yes";
c57f56b7 2071 else msg += " using_tags: No";
23329835 2072 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2073 return;
2074 }
a2f5fc01 2075 useTags = slfn.Contains(".tag");
c57f56b7 2076 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2077 else msg += " type: unknown file;";
a2f5fc01 2078 if (useTags) msg += " using_tags: Yes";
c57f56b7 2079 else msg += " using_tags: No";
23329835 2080 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2081}
2082
4e5c5506 2083//______________________________________________________________________________
2084void AliAnalysisAlien::EnablePackage(const char *package)
2085{
2086// Enables a par file supposed to exist in the current directory.
2087 TString pkg(package);
2088 pkg.ReplaceAll(".par", "");
2089 pkg += ".par";
2090 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2091 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2092 return;
2093 }
2094 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2095 Info("EnablePackage", "AliEn plugin will use .par packages");
2096 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2097 if (!fPackages) {
2098 fPackages = new TObjArray();
2099 fPackages->SetOwner();
2100 }
2101 fPackages->Add(new TObjString(pkg));
2102}
2103
3bdcb562 2104//______________________________________________________________________________
2105TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2106{
2107// Make a tree from files having the location specified in fFileForTestMode.
2108// Inspired from JF's CreateESDChain.
2109 if (fFileForTestMode.IsNull()) {
2110 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2111 return NULL;
2112 }
2113 if (gSystem->AccessPathName(fFileForTestMode)) {
2114 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2115 return NULL;
2116 }
2117 // Open the file
2118 ifstream in;
2119 in.open(fFileForTestMode);
2120 Int_t count = 0;
2121 // Read the input list of files and add them to the chain
2122 TString line;
2123 TChain *chain = new TChain(treeName);
2124 while (in.good())
2125 {
2126 in >> line;
2127 if (line.IsNull()) continue;
2128 if (count++ == fNtestFiles) break;
2129 TString esdFile(line);
2130 TFile *file = TFile::Open(esdFile);
2131 if (file) {
2132 if (!file->IsZombie()) chain->Add(esdFile);
2133 file->Close();
2134 } else {
2135 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2136 }
2137 }
2138 in.close();
2139 if (!chain->GetListOfFiles()->GetEntries()) {
2140 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2141 delete chain;
2142 return NULL;
2143 }
2144// chain->ls();
2145 return chain;
2146}
2147
d2a409b2 2148//______________________________________________________________________________
2149const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2150{
2151// Get job status for all jobs with jobid>jobidstart.
2152 static char mstatus[20];
2153 mstatus[0] = '\0';
2154 nrunning = 0;
2155 nwaiting = 0;
2156 nerror = 0;
2157 ndone = 0;
2158 TGridJobStatusList *list = gGrid->Ps("");
2159 if (!list) return mstatus;
2160 Int_t nentries = list->GetSize();
2161 TGridJobStatus *status;
2162 Int_t pid;
2163 for (Int_t ijob=0; ijob<nentries; ijob++) {
2164 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2165 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2166 if (pid<jobidstart) continue;
2167 if (pid == lastid) {
8ddad121 2168 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2169 }
2170 switch (status->GetStatus()) {
2171 case TGridJobStatus::kWAITING:
2172 nwaiting++; break;
2173 case TGridJobStatus::kRUNNING:
2174 nrunning++; break;
2175 case TGridJobStatus::kABORTED:
2176 case TGridJobStatus::kFAIL:
2177 case TGridJobStatus::kUNKNOWN:
2178 nerror++; break;
2179 case TGridJobStatus::kDONE:
2180 ndone++;
2181 }
2182 }
2183 list->Delete();
2184 delete list;
2185 return mstatus;
2186}
2187
c57f56b7 2188//______________________________________________________________________________
2189Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2190{
2191// Returns true if file is a collection. Functionality duplicated from
2192// TAlien::Type() because we don't want to directly depend on TAlien.
2193 if (!gGrid) {
2194 Error("IsCollection", "No connection to grid");
2195 return kFALSE;
2196 }
2197 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2198 if (!res) return kFALSE;
2199 const char* typeStr = res->GetKey(0, "type");
2200 if (!typeStr || !strlen(typeStr)) return kFALSE;
2201 if (!strcmp(typeStr, "collection")) return kTRUE;
2202 delete res;
2203 return kFALSE;
2204}
2205
fe2d7fc2 2206//______________________________________________________________________________
2207Bool_t AliAnalysisAlien::IsSingleOutput() const
2208{
2209// Check if single-ouput option is on.
2210 return (!fOutputSingle.IsNull());
2211}
2212
16a4353c 2213//______________________________________________________________________________
2214void AliAnalysisAlien::Print(Option_t *) const
2215{
2216// Print current plugin settings.
84fcd93f 2217 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2218 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2219 if (mgr && mgr->IsProofMode()) {
2220 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2221 if (TestBit(AliAnalysisGrid::kTest))
2222 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2223 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2224 if (!fProofDataSet.IsNull())
2225 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2226 if (fProofReset==1)
2227 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2228 if (fProofReset>1)
2229 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2230 if (!fRootVersionForProof.IsNull())
2231 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2232 else
2233 printf("= ROOT version requested________________________ default\n");
2234 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2235 if (!fAliRootMode.IsNull())
2236 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2237 if (fNproofWorkers)
2238 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2239 if (fNproofWorkersPerSlave)
2240 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2241 if (TestSpecialBit(kClearPackages))
2242 printf("= ClearPackages requested...\n");
2243 if (fIncludePath.Data())
2244 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2245 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2246 if (fPackages && fPackages->GetEntries()) {
2247 TIter next(fPackages);
2248 TObject *obj;
2249 TString list;
2250 while ((obj=next())) list += obj->GetName();
2251 printf("= Par files to be used: ________________________ %s\n", list.Data());
2252 }
2253 if (TestSpecialBit(kProofConnectGrid))
2254 printf("= Requested PROOF connection to grid\n");
2255 return;
2256 }
e1c22e21 2257 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2258 if (fOverwriteMode) {
2259 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2260 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2261 }
348be253 2262 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2263 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2264 printf("= Production mode:______________________________ %d\n", fProductionMode);
2265 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2266 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2267 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2268 if (fUser.Length())
84fcd93f 2269 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2270 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2271 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2272 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2273 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2274 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2275 if (fRunNumbers.Length())
84fcd93f 2276 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2277 if (fRunRange[0])
74013535 2278 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2279 if (!fRunRange[0] && !fRunNumbers.Length()) {
2280 TIter next(fInputFiles);
2281 TObject *obj;
2282 TString list;
2283 while ((obj=next())) list += obj->GetName();
84fcd93f 2284 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2285 }
2286 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2287 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2288 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2289 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2290 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 2291 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2292 printf("=====================================================================\n");
2293 printf("= Job price: ___________________________________ %d\n", fPrice);
2294 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2295 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2296 if (fMaxInitFailed>0)
84fcd93f 2297 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2298 if (fMasterResubmitThreshold>0)
84fcd93f 2299 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2300 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2301 if (fNrunsPerMaster>0)
84fcd93f 2302 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2303 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2304 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2305 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2306 if (fArguments.Length())
84fcd93f 2307 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2308 if (fExecutableArgs.Length())
2309 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2310 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2311 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2312 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2313 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2314 if (fDatasetName)
84fcd93f 2315 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2316 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2317 if (fIncludePath.Data())
84fcd93f 2318 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2319 if (fCloseSE.Length())
84fcd93f 2320 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2321 if (fFriendChainName.Length())
84fcd93f 2322 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2323 if (fPackages && fPackages->GetEntries()) {
16a4353c 2324 TIter next(fPackages);
2325 TObject *obj;
2326 TString list;
2327 while ((obj=next())) list += obj->GetName();
84fcd93f 2328 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2329 }
2330}
2331
c57f56b7 2332//______________________________________________________________________________
2333void AliAnalysisAlien::SetDefaults()
2334{
2335// Set default values for everything. What cannot be filled will be left empty.
2336 if (fGridJDL) delete fGridJDL;
2337 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2338 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2339 fPrice = 1;
2340 fTTL = 30000;
2341 fSplitMaxInputFileNumber = 100;
2342 fMaxInitFailed = 0;
2343 fMasterResubmitThreshold = 0;
bb885a9e 2344 fNtestFiles = 10;
149d288c 2345 fNreplicas = 2;
a8739e8a 2346 fRunRange[0] = 0;
2347 fRunRange[1] = 0;
8e3bd06a 2348 fRunPrefix = "%d";
319593fb 2349 fNrunsPerMaster = 1;
16a4353c 2350 fMaxMergeFiles = 100;
c57f56b7 2351 fRunNumbers = "";
2352 fExecutable = "analysis.sh";
0a1c1f7f 2353 fExecutableCommand = "root -b -q";
c57f56b7 2354 fArguments = "";
631c0b05 2355 fExecutableArgs = "";
c57f56b7 2356 fAnalysisMacro = "myAnalysis.C";
2357 fAnalysisSource = "";
2358 fAdditionalLibs = "";
2359 fSplitMode = "se";
2360 fAPIVersion = "";
2361 fROOTVersion = "";
2362 fAliROOTVersion = "";
2363 fUser = ""; // Your alien user name
2364 fGridWorkingDir = "";
2365 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2366 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2367 fFriendChainName = "";
c57f56b7 2368 fGridOutputDir = "output";
b83f84f0 2369 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2370 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2371 fInputFormat = "xml-single";
2372 fJDLName = "analysis.jdl";
c6cb3634 2373 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2374 fMergeExcludes = "";
0f389141 2375 fMergeViaJDL = 0;
348be253 2376 SetUseCopy(kTRUE);
2377 SetCheckCopy(kTRUE);
149d288c 2378 SetDefaultOutputs(kTRUE);
e1c22e21 2379 fOverwriteMode = 1;
c57f56b7 2380}
2381
0f389141 2382//______________________________________________________________________________
b3e07543 2383Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2384{
b3e07543 2385// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2386 // First check if the result is already in the output directory.
2387 if (FileExists(Form("%s/%s",aliendir,filename))) {
2388 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2389 return kFALSE;
2390 }
b3e07543 2391 // Now check the last stage done.
2392 Int_t stage = 0;
2393 while (1) {
2394 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2395 stage++;
2396 }
2397 // Next stage of merging
2398 stage++;
2399 TString pattern = "*root_archive.zip";
2400 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2401 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2402 if (res) delete res;
2403 // Write standard output to file
2404 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2405 // Count the number of files inside
2406 ifstream ifile;
2407 ifile.open(Form("Stage_%d.xml",stage));
2408 if (!ifile.good()) {
2409 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2410 return kFALSE;
b3e07543 2411 }
2412 TString line;
2413 Int_t nfiles = 0;
2414 while (!ifile.eof()) {
2415 ifile >> line;
2416 if (line.Contains("/event")) nfiles++;
7c2cd90a 2417 }
b3e07543 2418 ifile.close();
2419 if (!nfiles) {
2420 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2421 return kFALSE;
2422 } else {
2423 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2424 }
b3e07543 2425 // Copy the file in the output directory
2426 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
2427 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2428 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2429 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2430 // Check if this is the last stage to be done.
2431 Bool_t laststage = (nfiles<nperchunk);
2432 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2433 if (laststage) {
2434 printf("### Submiting final merging stage %d\n", stage);
2435 TString finalJDL = jdl;
2436 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2437 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2438 Int_t jobId = SubmitSingleJob(query);
2439 if (!jobId) return kFALSE;
2440 } else {
2441 printf("### Submiting merging stage %d\n", stage);
03335324 2442 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2443 Int_t jobId = SubmitSingleJob(query);
2444 if (!jobId) return kFALSE;
7c2cd90a 2445 }
b3e07543 2446 return kTRUE;
2447}
7c2cd90a 2448
105631b5 2449//______________________________________________________________________________
2450AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2451{
2452// Loat the analysis manager from a file.
2453 TFile *file = TFile::Open(fname);
2454 if (!file) {
2455 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2456 return 0;
2457 }
2458 TIter nextkey(file->GetListOfKeys());
2459 AliAnalysisManager *mgr = 0;
2460 TKey *key;
2461 while ((key=(TKey*)nextkey())) {
2462 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2463 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2464 }
2465 if (!mgr)
2466 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2467 return mgr;
2468}
2469
7c2cd90a 2470//______________________________________________________________________________
2471Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2472{
2473// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2474 if (!gGrid) return 0;
2475 printf("=> %s ------> ",query);
2476 TGridResult *res = gGrid->Command(query);
2477 if (!res) return 0;
2478 TString jobId = res->GetKey(0,"jobId");
2479 delete res;
2480 if (jobId.IsNull()) {
2481 printf("submission failed. Reason:\n");
2482 gGrid->Stdout();
2483 gGrid->Stderr();
2484 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2485 return 0;
2486 }
2487 printf(" Job id: %s\n", jobId.Data());
2488 return atoi(jobId);
2489}
2490
2491//______________________________________________________________________________
b3e07543 2492Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2493{
b3e07543 2494// Merge given output files from basedir. Basedir can be an alien output directory
2495// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2496// files in a group (ignored for xml input). Merging can be done in stages:
2497// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2498// stage=1 : works with an xml of all root_archive.zip in the output directory
2499// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2500 TString outputFile = output;
0f389141 2501 TString command;
a2f5fc01 2502 TString outputChunk;
2503 TString previousChunk = "";
b3e07543 2504 TObjArray *listoffiles = new TObjArray();
2505// listoffiles->SetOwner();
a2f5fc01 2506 Int_t countChunk = 0;
2507 Int_t countZero = nmaxmerge;
0f389141 2508 Bool_t merged = kTRUE;
a2f5fc01 2509 Int_t index = outputFile.Index("@");
2510 if (index > 0) outputFile.Remove(index);
7c2cd90a 2511 TString inputFile = outputFile;
b3e07543 2512 TString sbasedir = basedir;
2513 if (sbasedir.Contains(".xml")) {
2514 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2515 nmaxmerge = 9999999;
2516 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2517 if (!coll) {
2518 ::Error("MergeOutput", "Input XML collection empty.");
2519 return kFALSE;
2520 }
2521 // Iterate grid collection
2522 while (coll->Next()) {
2523 TString fname = gSystem->DirName(coll->GetTURL());
2524 fname += "/";
2525 fname += inputFile;
2526 listoffiles->Add(new TNamed(fname.Data(),""));
2527 }
2528 } else {
2529 command = Form("find %s/ *%s", basedir, inputFile.Data());
2530 printf("command: %s\n", command.Data());
2531 TGridResult *res = gGrid->Command(command);
2532 if (!res) {
2533 ::Error("MergeOutput","No result for the find command\n");
2534 delete listoffiles;
2535 return kFALSE;
2536 }
2537 TIter nextmap(res);
2538 TMap *map = 0;
2539 while ((map=(TMap*)nextmap())) {
2540 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2541 if (!objs || !objs->GetString().Length()) {
2542 // Nothing found - skip this output
2543 delete res;
2544 delete listoffiles;
2545 return kFALSE;
2546 }
2547 listoffiles->Add(new TNamed(objs->GetName(),""));
2548 }
2549 delete res;
2550 }
2551 if (!listoffiles->GetEntries()) {
7c2cd90a 2552 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2553 delete listoffiles;
0f389141 2554 return kFALSE;
2555 }
2556
2557 TFileMerger *fm = 0;
b3e07543 2558 TIter next0(listoffiles);
2559 TObjArray *listoffilestmp = new TObjArray();
2560 listoffilestmp->SetOwner();
2561 TObject *nextfile;
2562 TString snextfile;
2563 // Keep only the files at upper level
2564 Int_t countChar = 0;
2565 while ((nextfile=next0())) {
2566 snextfile = nextfile->GetName();
2567 Int_t crtCount = snextfile.CountChar('/');
2568 if (nextfile == listoffiles->First()) countChar = crtCount;
2569 if (crtCount < countChar) countChar = crtCount;
2570 }
2571 next0.Reset();
2572 while ((nextfile=next0())) {
2573 snextfile = nextfile->GetName();
2574 Int_t crtCount = snextfile.CountChar('/');
2575 if (crtCount > countChar) {
2576 delete nextfile;
2577 continue;
2578 }
2579 listoffilestmp->Add(nextfile);
2580 }
2581 delete listoffiles;
2582 listoffiles = listoffilestmp; // Now contains 'good' files
2583 listoffiles->Print();
2584 TIter next(listoffiles);
7c2cd90a 2585 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2586 outputChunk = outputFile;
2587 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2588 // Check for existent temporary merge files
e1c22e21 2589 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2590 // Preserve old merging functionality for stage 0.
2591 if (stage==0) {
2592 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2593 while (1) {
2594 // Skip as many input files as in a chunk
2ea55496 2595 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2596 nextfile = next();
2597 if (!nextfile) {
2ea55496 2598 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2599 delete listoffiles;
2ea55496 2600 return kFALSE;
2601 }
b3e07543 2602 snextfile = nextfile->GetName();
7c2cd90a 2603 }
2604 outputChunk = outputFile;
2605 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2606 countChunk++;
2607 if (gSystem->AccessPathName(outputChunk)) continue;
2608 // Merged file with chunks up to <countChunk> found
2609 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2610 previousChunk = outputChunk;
2611 break;
2612 }
2613 }
2614 countZero = nmaxmerge;
2615
b3e07543 2616 while ((nextfile=next())) {
2617 snextfile = nextfile->GetName();
2ea55496 2618 // Loop 'find' results and get next LFN
7c2cd90a 2619 if (countZero == nmaxmerge) {
2620 // First file in chunk - create file merger and add previous chunk if any.
2621 fm = new TFileMerger(kFALSE);
2622 fm->SetFastMethod(kTRUE);
2623 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2624 outputChunk = outputFile;
2625 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2626 }
2627 // If last file found, put merged results in the output file
b3e07543 2628 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2629 // Add file to be merged and decrement chunk counter.
b3e07543 2630 fm->AddFile(snextfile);
7c2cd90a 2631 countZero--;
b3e07543 2632 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2633 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2634 // Nothing found - skip this output
2635 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2636 merged = kFALSE;
2637 break;
7c2cd90a 2638 }
2639 fm->OutputFile(outputChunk);
2640 // Merge the outputs, then go to next chunk
2641 if (!fm->Merge()) {
2642 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2643 merged = kFALSE;
2644 break;
7c2cd90a 2645 } else {
2646 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2647 gSystem->Unlink(previousChunk);
2648 }
b3e07543 2649 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2650 countChunk++;
2651 countZero = nmaxmerge;
2652 previousChunk = outputChunk;
0f389141 2653 }
0f389141 2654 }
b3e07543 2655 delete listoffiles;
2656 delete fm;
7c2cd90a 2657 return merged;
2658 }
2659 // Merging stage different than 0.
2660 // Move to the begining of the requested chunk.
7c2cd90a 2661 fm = new TFileMerger(kFALSE);
2662 fm->SetFastMethod(kTRUE);
b3e07543 2663 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2664 delete listoffiles;
7c2cd90a 2665 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2666 // Nothing found - skip this output
2667 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2668 delete fm;
2669 return kFALSE;
2670 }
b3e07543 2671 fm->OutputFile(outputFile);
7c2cd90a 2672 // Merge the outputs
2673 if (!fm->Merge()) {
2674 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2675 delete fm;
2676 return kFALSE;
2677 } else {
b3e07543 2678 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2679 }
7c2cd90a 2680 delete fm;
2681 return kTRUE;
0f389141 2682}
2683
c57f56b7 2684//______________________________________________________________________________
2685Bool_t AliAnalysisAlien::MergeOutputs()
2686{
2687// Merge analysis outputs existing in the AliEn space.
2688 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2689 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2690 if (!Connect()) {
2691 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2692 return kFALSE;
0f389141 2693 }
c9e8f7fd 2694 if (fMergeViaJDL) {
2695 if (!TestBit(AliAnalysisGrid::kMerge)) {
2696 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2697 return kFALSE;
2698 }
2699 if (fProductionMode) {
2700 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2701 return kFALSE;
2702 }
0f389141 2703 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2704 if (!SubmitMerging()) return kFALSE;
0f389141 2705 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2706 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2707 return kFALSE;
c57f56b7 2708 }
2709 // Get the output path
dc030a24 2710 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2711 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2712 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2713 return kFALSE;
2714 }
2715 if (!fOutputFiles.Length()) {
2716 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2717 return kFALSE;
0f389141 2718 }
2719 // Check if fast read option was requested
e1c22e21 2720 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2721 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2722 if (fFastReadOption) {
d3339be3 2723 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2724 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2725 gEnv->SetValue("XNet.ConnectTimeout",50);
2726 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2727 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2728 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2729 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2730 }
e8b839ab 2731 // Make sure we change the temporary directory
2732 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2733 // Set temporary compilation directory to current one
2734 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2735 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2736 TIter next(list);
2737 TObjString *str;
a2f5fc01 2738 TString outputFile;
c57f56b7 2739 Bool_t merged = kTRUE;
2740 while((str=(TObjString*)next())) {
a2f5fc01 2741 outputFile = str->GetString();
2742 Int_t index = outputFile.Index("@");
2743 if (index > 0) outputFile.Remove(index);
2744 TString outputChunk = outputFile;
2745 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2746 // Skip already merged outputs
a2f5fc01 2747 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2748 if (fOverwriteMode) {
a2f5fc01 2749 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2750 gSystem->Unlink(outputFile);
2751 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2752 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2753 outputChunk.Data());
2754 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2755 }
2756 } else {
a2f5fc01 2757 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2758 continue;
2759 }
2760 } else {
a2f5fc01 2761 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2762 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2763 outputChunk.Data());
2764 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2765 }
2766 }
bb885a9e 2767 if (fMergeExcludes.Length() &&
a2f5fc01 2768 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2769 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2770 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2771 if (!merged) {
2772 Error("MergeOutputs", "Terminate() will NOT be executed");
2773 return kFALSE;
ff07ec61 2774 }
2775 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2776 if (fileOpened) fileOpened->Close();
c57f56b7 2777 }
0f389141 2778 return kTRUE;
c57f56b7 2779}
2780
bb885a9e 2781//______________________________________________________________________________
2782void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2783{
2784// Use the output files connected to output containers from the analysis manager
2785// rather than the files defined by SetOutputFiles
2786 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2787 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2788 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2789}
2790
149d288c 2791//______________________________________________________________________________
2792void AliAnalysisAlien::SetOutputFiles(const char *list)
2793{
2794// Manually set the output files list.
2795// Removes duplicates. Not allowed if default outputs are not disabled.
2796 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2797 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2798 return;
2799 }
2800 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2801 fOutputFiles = "";
2802 TString slist = list;
2803 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2804 TObjArray *arr = slist.Tokenize(" ");
2805 TObjString *os;
2806 TIter next(arr);
2807 TString sout;
2808 while ((os=(TObjString*)next())) {
2809 sout = os->GetString();
2810 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2811 if (fOutputFiles.Contains(sout)) continue;
2812 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2813 fOutputFiles += sout;
2814 }
2815 delete arr;
f790bc1b 2816}
149d288c 2817
2818//______________________________________________________________________________
2819void AliAnalysisAlien::SetOutputArchive(const char *list)
2820{
2821// Manually set the output archive list. Free text - you are on your own...
2822// Not allowed if default outputs are not disabled.
2823 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2824 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2825 return;
2826 }
2827 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2828 fOutputArchive = list;
2829}
2830
2831//______________________________________________________________________________
2832void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2833{
2834// Setting a prefered output SE is not allowed anymore.
2835 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2836}
2837
28ce6c9c 2838//______________________________________________________________________________
2839void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2840{
2841// Set some PROOF special parameter.
2842 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2843 if (pair) {
2844 TObject *old = pair->Key();
2845 TObject *val = pair->Value();
2846 fProofParam.Remove(old);
2847 delete old;
2848 delete val;
2849 }
2850 fProofParam.Add(new TObjString(pname), new TObjString(value));
2851}
2852
2853//______________________________________________________________________________
2854const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2855{
2856// Returns a special PROOF parameter.
2857 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2858 if (!pair) return 0;
2859 return pair->Value()->GetName();
2860}
2861
5513444a 2862//______________________________________________________________________________
2863Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2864{
2865// Start remote grid analysis.
3bdcb562 2866 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2867 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2868 if (!mgr || !mgr->IsInitialized()) {
2869 Error("StartAnalysis", "You need an initialized analysis manager for this");
2870 return kFALSE;
2871 }
2872 // Are we in PROOF mode ?
2873 if (mgr->IsProofMode()) {
baa34407 2874 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2875 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2876 if (fProofCluster.IsNull()) {
2877 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2878 return kFALSE;
2879 }
2880 if (fProofDataSet.IsNull() && !testMode) {
2881 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2882 return kFALSE;
2883 }
2884 // Set the needed environment
2885 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2886 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2887 if (fProofReset && !testMode) {
2888 if (fProofReset==1) {
2889 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2890 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2891 } else {
2892 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2893 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2894 }
2895 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2896 return kFALSE;
2897 }
baa34407 2898
2899 if (!testMode) {
2900 // Check if there is an old active session
2901 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2902 if (nsessions) {
2903 Error("StartAnalysis","You have to reset your old session first\n");
2904 return kFALSE;
2905 }
53911ca6 2906 }
3bdcb562 2907 // Do we need to change the ROOT version ? The success of this cannot be checked.
2908 if (!fRootVersionForProof.IsNull() && !testMode) {
2909 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2910 fProofCluster.Data(), fRootVersionForProof.Data()));
2911 }
2912 // Connect to PROOF and check the status
2913 Long_t proof = 0;
27734f0e 2914 TString sworkers;
2915 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2916 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2917 if (!testMode) {
27734f0e 2918 if (!sworkers.IsNull())
2919 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2920 else
2921 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2922 } else {
2923 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2924 if (!proof) {
2925 Error("StartAnalysis", "Could not start PROOF in test mode");
2926 return kFALSE;
2927 }
2928 }
2929 if (!proof) {
2930 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2931 return kFALSE;
2932 }
27734f0e 2933 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2934 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2935 // Set proof special parameters if any
2936 TIter nextpp(&fProofParam);
2937 TObject *proofparam;
2938 while ((proofparam=nextpp())) {
2939 TString svalue = GetProofParameter(proofparam->GetName());
2940 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
2941 }
3bdcb562 2942 // Is dataset existing ?
2943 if (!testMode) {
2944 TString dataset = fProofDataSet;
2945 Int_t index = dataset.Index("#");
2946 if (index>=0) dataset.Remove(index);
830acc4c 2947// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2948// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2949// return kFALSE;
2950// }
2951// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2952 }
2953 // Is ClearPackages() needed ?
2954 if (TestSpecialBit(kClearPackages)) {
2955 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2956 gROOT->ProcessLine("gProof->ClearPackages();");
2957 }
2958 // Is a given aliroot mode requested ?
2959 TList optionsList;
1ac846e9 2960 TString parLibs;
830acc4c 2961 if (!fAliRootMode.IsNull()) {
3bdcb562 2962 TString alirootMode = fAliRootMode;
2963 if (alirootMode == "default") alirootMode = "";
2964 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2965 optionsList.SetOwner();
2966 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2967 // Check the additional libs to be loaded
2968 TString extraLibs;
1ac846e9 2969 Bool_t parMode = kFALSE;
dc1ffbcb 2970 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 2971 // Parse the extra libs for .so
2972 if (fAdditionalLibs.Length()) {
2973 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2974 TIter next(list);
2975 TObjString *str;
1ac846e9 2976 while((str=(TObjString*)next())) {
2977 if (str->GetString().Contains(".so")) {
2978 if (parMode) {
2979 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2980 break;
2981 }
2982 TString stmp = str->GetName();
2983 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2984 stmp.ReplaceAll(".so","");
2985 if (!extraLibs.IsNull()) extraLibs += ":";
2986 extraLibs += stmp;
2987 continue;
2988 }
2989 if (str->GetString().Contains(".par")) {
2990 // The first par file found in the list will not allow any further .so
2991 parMode = kTRUE;
2992 if (!parLibs.IsNull()) parLibs += ":";
2993 parLibs += str->GetName();
2994 continue;
2995 }
3bdcb562 2996 }
2997 if (list) delete list;
2998 }
dc030a24 2999 if (!extraLibs.IsNull()) {
3000 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3001 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3002 }
830acc4c 3003 // Check extra includes
3004 if (!fIncludePath.IsNull()) {
3005 TString includePath = fIncludePath;
3006 includePath.ReplaceAll(" ",":");
dc030a24 3007 includePath.ReplaceAll("$ALICE_ROOT/","");
3008 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3009 includePath.ReplaceAll("-I","");
dc030a24 3010 includePath.Remove(TString::kTrailing, ':');
830acc4c 3011 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3012 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3013 }
27734f0e 3014 // Check if connection to grid is requested
3015 if (TestSpecialBit(kProofConnectGrid))
3016 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3017 // Enable AliRoot par
3018 if (testMode) {
3019 // Enable proof lite package
3020 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3021 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3022 TNamed *obj = (TNamed*)optionsList.At(i);
3023 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3024 }
3025 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3026 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3027 Info("StartAnalysis", "AliRootProofLite enabled");
3028 } else {
3029 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3030 return kFALSE;
3031 }
3032 } else {
dc030a24 3033 if ( ! fAliROOTVersion.IsNull() ) {
3034 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3035 fAliROOTVersion.Data(), &optionsList))) {
3036 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3037 return kFALSE;
3038 }
3039 }
27734f0e 3040 }
1ac846e9 3041 // Enable first par files from fAdditionalLibs
3042 if (!parLibs.IsNull()) {
3043 TObjArray *list = parLibs.Tokenize(":");
3044 TIter next(list);
3045 TObjString *package;
3046 while((package=(TObjString*)next())) {
40501bb7 3047 TString spkg = package->GetName();
3048 spkg.ReplaceAll(".par", "");
3049 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3050 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3051 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3052 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3053 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3054 return kFALSE;
3055 }
3056 } else {
3057 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3058 return kFALSE;
3059 }
3060 }
3061 if (list) delete list;
3062 }
3bdcb562 3063 } else {
3064 if (fAdditionalLibs.Contains(".so") && !testMode) {
3065 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3066 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3067 return kFALSE;
3068 }
3069 }
3070 // Enable par files if requested
3071 if (fPackages && fPackages->GetEntries()) {
3072 TIter next(fPackages);
3073 TObject *package;
3074 while ((package=next())) {
1ac846e9 3075 // Skip packages already enabled
3076 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3077 TString spkg = package->GetName();
3078 spkg.ReplaceAll(".par", "");
3079 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3080 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3081 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3082 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3083 return kFALSE;
3084 }
3085 } else {
3086 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3087 return kFALSE;
3088 }
3089 }
3090 }
3091 // Do we need to load analysis source files ?
3092 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3093 if (fAnalysisSource.Length()) {
3094 TObjArray *list = fAnalysisSource.Tokenize(" ");
3095 TIter next(list);
3096 TObjString *str;
3097 while((str=(TObjString*)next())) {
3098 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3099 }
3bdcb562 3100 if (list) delete list;
3101 }
830acc4c 3102 if (testMode) {
3103 // Register dataset to proof lite.
3104 if (fFileForTestMode.IsNull()) {
3105 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3106 return kFALSE;
3107 }
3108 if (gSystem->AccessPathName(fFileForTestMode)) {
3109 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3110 return kFALSE;
3111 }
3112 TFileCollection *coll = new TFileCollection();
3113 coll->AddFromFile(fFileForTestMode);
8ddad121 3114 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3115 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3116 }
3bdcb562 3117 return kTRUE;
3118 }
c57f56b7 3119
43da816a 3120 // Check if output files have to be taken from the analysis manager
3121 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3122 // Add output files and AOD files
3123 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3124 // Add extra files registered to the analysis manager
f790bc1b 3125 TString extra = GetListOfFiles("ext");
3126 if (!extra.IsNull()) {
149d288c 3127 extra.ReplaceAll(".root", "*.root");
f790bc1b 3128 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3129 fOutputFiles += extra;
84fcd93f 3130 }
149d288c 3131 // Compose the output archive.
b83f84f0 3132 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3133 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3134 }
f7b1cbc2 3135// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3136 if (TestBit(AliAnalysisGrid::kOffline)) {
3137 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3138 \n there nor any job run. You can revise the JDL and analysis \
3139 \n macro then run the same in \"submit\" mode.");
3140 } else if (TestBit(AliAnalysisGrid::kTest)) {
3141 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3142 \n dataset.");
3143 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3144 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3145 \n space and job submitted.");
3146 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3147 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3148 if (fMergeViaJDL) CheckInputData();
5513444a 3149 return kTRUE;
c57f56b7 3150 } else {
3151 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3152 }
3153
348be253 3154 Print();
c57f56b7 3155 if (!Connect()) {
3156 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3157 return kFALSE;
16a4353c 3158 }
d3b18c4c 3159 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3160 if (!CheckInputData()) {
3161 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3162 return kFALSE;
c57f56b7 3163 }
d3339be3 3164 if (!CreateDataset(fDataPattern)) {
3165 TString serror;
3166 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3167 if (fRunNumbers.Length()) serror = "run numbers";
3168 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3169 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3170 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3171 return kFALSE;
3172 }
21ca8e59 3173 WriteAnalysisFile();
c57f56b7 3174 WriteAnalysisMacro();
3175 WriteExecutable();
3176 WriteValidationScript();
0f389141 3177 if (fMergeViaJDL) {
3178 WriteMergingMacro();
3179 WriteMergeExecutable();
3180 WriteValidationScript(kTRUE);
3181 }
5513444a 3182 if (!CreateJDL()) return kFALSE;
3183 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3184 if (testMode) {
c57f56b7 3185 // Locally testing the analysis
3186 Info("StartAnalysis", "\n_______________________________________________________________________ \
3187 \n Running analysis script in a daughter shell as on a worker node \
3188 \n_______________________________________________________________________");
149d288c 3189 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3190 TIter next(list);
3191 TObjString *str;
a2f5fc01 3192 TString outputFile;
c57f56b7 3193 while((str=(TObjString*)next())) {
a2f5fc01 3194 outputFile = str->GetString();
3195 Int_t index = outputFile.Index("@");
3196 if (index > 0) outputFile.Remove(index);
3197 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3198 }
3199 delete list;
3200 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3201 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3202// gSystem->Exec("cat stdout");
5513444a 3203 return kFALSE;
c57f56b7 3204 }
5513444a 3205 // Check if submitting is managed by LPM manager
a3e84053 3206 if (fProductionMode) {
5513444a 3207 TString prodfile = fJDLName;
3208 prodfile.ReplaceAll(".jdl", ".prod");
3209 WriteProductionFile(prodfile);
3210 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3211 return kFALSE;
3212 }
a8739e8a 3213 // Submit AliEn job(s)
d2a409b2 3214 gGrid->Cd(fGridOutputDir);
a8739e8a 3215 TGridResult *res;
c57f56b7 3216 TString jobID = "";
d2a409b2 3217 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3218 // Submit a given xml or a set of runs
a8739e8a 3219 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3220 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3221 if (res) {
3222 const char *cjobId = res->GetKey(0,"jobId");
3223 if (!cjobId) {
a03be957 3224 gGrid->Stdout();
3225 gGrid->Stderr();
a8739e8a 3226 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3227 return kFALSE;
a8739e8a 3228 } else {
3229 Info("StartAnalysis", "\n_______________________________________________________________________ \
3230 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3231 \n_______________________________________________________________________",
3232 fJDLName.Data(), cjobId);
3233 jobID = cjobId;
3234 }
3235 delete res;
a03be957 3236 } else {
3237 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3238 return kFALSE;
a8739e8a 3239 }
3240 } else {
d2a409b2 3241 // Submit for a range of enumeration of runs.
a03be957 3242 if (!Submit()) return kFALSE;
c57f56b7 3243 }
a8739e8a 3244
c57f56b7 3245 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3246 \n You may exit at any time and terminate the job later using the option <terminate> \
3247 \n ##################################################################################", jobID.Data());
bb885a9e 3248 gSystem->Exec("aliensh");
5513444a 3249 return kTRUE;
c57f56b7 3250}
3251
f790bc1b 3252//______________________________________________________________________________
3253const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3254{
3255// Get a comma-separated list of output files of the requested type.
3256// Type can be (case unsensitive):
3257// aod - list of aod files (std, extensions and filters)
3258// out - list of output files connected to containers (but not aod's or extras)
3259// ext - list of extra files registered to the manager
3260// ter - list of files produced in terminate
3261 static TString files;
3262 files = "";
3263 TString stype = type;
3264 stype.ToLower();
3265 TString aodfiles, extra;
3266 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3267 if (!mgr) {
3268 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3269 return files.Data();
3270 }
3271 if (mgr->GetOutputEventHandler()) {
3272 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3273 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3274 if (!extraaod.IsNull()) {
3275 aodfiles += ",";
3276 aodfiles += extraaod;
3277 }
3278 }
3279 if (stype.Contains("aod")) {
3280 files = aodfiles;
3281 if (stype == "aod") return files.Data();
3282 }
3283 // Add output files that are not in the list of AOD files
3284 TString outputfiles = "";
3285 TIter next(mgr->GetOutputs());
3286 AliAnalysisDataContainer *output;
3287 const char *filename = 0;
3288 while ((output=(AliAnalysisDataContainer*)next())) {
3289 filename = output->GetFileName();
3290 if (!(strcmp(filename, "default"))) continue;
3291 if (outputfiles.Contains(filename)) continue;
3292 if (aodfiles.Contains(filename)) continue;
3293 if (!outputfiles.IsNull()) outputfiles += ",";
3294 outputfiles += filename;
3295 }
3296 if (stype.Contains("out")) {
3297 if (!files.IsNull()) files += ",";
3298 files += outputfiles;
3299 if (stype == "out") return files.Data();
3300 }
3301 // Add extra files registered to the analysis manager
3302 TString sextra;
3303 extra = mgr->GetExtraFiles();
3304 if (!extra.IsNull()) {
3305 extra.Strip();
3306 extra.ReplaceAll(" ", ",");
3307 TObjArray *fextra = extra.Tokenize(",");
3308 TIter nextx(fextra);
3309 TObject *obj;
3310 while ((obj=nextx())) {
3311 if (aodfiles.Contains(obj->GetName())) continue;
3312 if (outputfiles.Contains(obj->GetName())) continue;
3313 if (sextra.Contains(obj->GetName())) continue;
3314 if (!sextra.IsNull()) sextra += ",";
3315 sextra += obj->GetName();
3316 }
3317 delete fextra;
3318 if (stype.Contains("ext")) {
3319 if (!files.IsNull()) files += ",";
3320 files += sextra;
3321 }
3322 }
3323 if (stype == "ext") return files.Data();
3324 TString termfiles;
3325 if (!fTerminateFiles.IsNull()) {
3326 fTerminateFiles.Strip();
3327 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3328 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3329 TIter nextx(fextra);
3330 TObject *obj;
3331 while ((obj=nextx())) {
3332 if (aodfiles.Contains(obj->GetName())) continue;
3333 if (outputfiles.Contains(obj->GetName())) continue;
3334 if (termfiles.Contains(obj->GetName())) continue;
3335 if (sextra.Contains(obj->GetName())) continue;
3336 if (!termfiles.IsNull()) termfiles += ",";
3337 termfiles += obj->GetName();
3338 }
3339 delete fextra;
3340 }
3341 if (stype.Contains("ter")) {
3342 if (!files.IsNull() && !termfiles.IsNull()) {
3343 files += ",";
3344 files += termfiles;
3345 }
3346 }
3347 return files.Data();
3348}
3349
d2a409b2 3350//______________________________________________________________________________
a03be957 3351Bool_t AliAnalysisAlien::Submit()
d2a409b2 3352{
3353// Submit all master jobs.
3354 Int_t nmasterjobs = fInputFiles->GetEntries();
3355 Long_t tshoot = gSystem->Now();
a03be957 3356 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3357 while (fNsubmitted < nmasterjobs) {
3358 Long_t now = gSystem->Now();
3359 if ((now-tshoot)>30000) {
3360 tshoot = now;
a03be957 3361 if (!SubmitNext()) return kFALSE;
d2a409b2 3362 }
3363 }
a03be957 3364 return kTRUE;
d2a409b2 3365}
3366
0f389141 3367//______________________________________________________________________________
a03be957 3368Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3369{
3370// Submit all merging jobs.
dc030a24 3371 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3372 gGrid->Cd(fGridOutputDir);
3373 TString mergeJDLName = fExecutable;
3374 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3375 if (!fInputFiles) {
3376 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3377 return kFALSE;
3378 }
0f389141 3379 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3380 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3381 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3382 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3383 if (fOutputToRunNo) {
3384 // The output directory is the run number
3385 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3386 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3387 } else {
17b4e9a4 3388 if (!fRunNumbers.Length() && !fRunRange[0]) {
3389 // The output directory is the grid outdir
3390 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3391 runOutDir = fGridOutputDir;
3392 } else {
3393 // The output directory is the master number in 3 digits format
3394 printf("### Submitting merging job for master <%03d>\n", i);
3395 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3396 }
7c2cd90a 3397 }
3398 // Check now the number of merging stages.
2b222f7c 3399 TObjArray *list = fOutputFiles.Tokenize(",");
3400 TIter next(list);
3401 TObjString *str;
3402 TString outputFile;
3403 while((str=(TObjString*)next())) {
3404 outputFile = str->GetString();
3405 Int_t index = outputFile.Index("@");
3406 if (index > 0) outputFile.Remove(index);
3407 if (!fMergeExcludes.Contains(outputFile)) break;
3408 }
3409 delete list;
b3e07543 3410 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3411 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3412 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3413 }
a03be957 3414 if (!ntosubmit) return kTRUE;
9f020952 3415 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3416 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3417 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3418 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3419 \n ################################################################################################################");
0f389141 3420 gSystem->Exec("aliensh");
a03be957 3421 return kTRUE;
0f389141 3422}
3423
d2a409b2 3424//______________________________________________________________________________
a03be957 3425Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3426{
08d5b699 3427// Submit next bunch of master jobs if the queue is free. The first master job is
3428// submitted right away, while the next will not be unless the previous was split.
3429// The plugin will not submit new master jobs if there are more that 500 jobs in
3430// waiting phase.
d2a409b2 3431 static Bool_t iscalled = kFALSE;
3432 static Int_t firstmaster = 0;
3433 static Int_t lastmaster = 0;
3434 static Int_t npermaster = 0;
a03be957 3435 if (iscalled) return kTRUE;
d2a409b2 3436 iscalled = kTRUE;
3437 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3438 Int_t ntosubmit = 0;
3439 TGridResult *res;
3440 TString jobID = "";
3bdcb562 3441 Int_t nmasterjobs = fInputFiles->GetEntries();
3442 if (!fNsubmitted) {
3443 ntosubmit = 1;
3444 if (!IsUseSubmitPolicy()) {
90d50a8c 3445 if (nmasterjobs>5)
3bdcb562 3446 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3447 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3448 ntosubmit = nmasterjobs;
3449 }
3450 } else {
d2a409b2 3451 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3452 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3453 // If last master not split, just return
a03be957 3454 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3455 // No more than 100 waiting jobs
08d5b699 3456 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3457 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3458 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3459 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3460 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3461 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3462 }
d2a409b2 3463 for (Int_t i=0; i<ntosubmit; i++) {
3464 // Submit for a range of enumeration of runs.
a03be957 3465 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3466 TString query;
cd11251e 3467 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3468 runOutDir.ReplaceAll(".xml", "");
3469 if (fOutputToRunNo)
3470 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3471 else
3472 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3473 printf("********* %s\n",query.Data());
d2a409b2 3474 res = gGrid->Command(query);
3475 if (res) {
98ca124f 3476 TString cjobId1 = res->GetKey(0,"jobId");
3477 if (!cjobId1.Length()) {
d2a409b2 3478 iscalled = kFALSE;
a03be957 3479 gGrid->Stdout();
3480 gGrid->Stderr();
3481 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3482 return kFALSE;
d2a409b2 3483 } else {
3484 Info("StartAnalysis", "\n_______________________________________________________________________ \
3485 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3486 \n_______________________________________________________________________",
98ca124f 3487 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3488 jobID += cjobId1;
3489 jobID += " ";
98ca124f 3490 lastmaster = cjobId1.Atoi();
d2a409b2 3491 if (!firstmaster) firstmaster = lastmaster;
3492 fNsubmitted++;
3493 }
3494 delete res;
a03be957 3495 } else {
3496 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3497 return kFALSE;
d2a409b2 3498 }
3499 }
3500 iscalled = kFALSE;
a03be957 3501 return kTRUE;
d2a409b2 3502}
3503
c57f56b7 3504//______________________________________________________________________________
3505void AliAnalysisAlien::WriteAnalysisFile()
3506{
f10e8481 3507// Write current analysis manager into the file <analysisFile>
3508 TString analysisFile = fExecutable;
3509 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3510 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3511 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3512 if (!mgr || !mgr->IsInitialized()) {
3513 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3514 return;
3515 }
3516 // Check analysis type
3517 TObject *handler;
3518 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3519 handler = (TObject*)mgr->GetInputEventHandler();
3520 if (handler) {
65eb22f8 3521 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3522 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3523 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3524 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3525 } else {
3526 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3527 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3528 }
c57f56b7 3529 }
3530 TDirectory *cdir = gDirectory;
f10e8481 3531 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3532 if (file) {
e16a394c 3533 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3534 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3535 // Unless merging makes no sense
3536 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3537 mgr->Write();
3538 delete file;
fe2d7fc2 3539 // Enable termination for local jobs
3540 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3541 }
3542 if (cdir) cdir->cd();
f10e8481 3543 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3544 }
3545 Bool_t copy = kTRUE;
d3b18c4c 3546 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3547 if (copy) {
3548 CdWork();
3549 TString workdir = gGrid->GetHomeDirectory();
3550 workdir += fGridWorkingDir;
d3b18c4c 3551 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3552 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3553 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
6c395669 3554 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3555 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3556 }
3557}
3558
3559//______________________________________________________________________________
3560void AliAnalysisAlien::WriteAnalysisMacro()
3561{
3562// Write the analysis macro that will steer the analysis in grid mode.
3563 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3564 ofstream out;
3565 out.open(fAnalysisMacro.Data(), ios::out);
3566 if (!out.good()) {
3567 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3568 return;
3569 }
5b273635 3570 Bool_t hasSTEERBase = kFALSE;
3571 Bool_t hasESD = kFALSE;
3572 Bool_t hasAOD = kFALSE;
3573 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3574 Bool_t hasOADB = kFALSE;
5b273635 3575 Bool_t hasANALYSISalice = kFALSE;
3576 Bool_t hasCORRFW = kFALSE;
c57f56b7 3577 TString func = fAnalysisMacro;
3578 TString type = "ESD";
3579 TString comment = "// Analysis using ";
c85cfc0f 3580 if (IsUseMCchain()) {
3581 type = "MC";
3582 comment += "MC";
3583 } else {
3584 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3585 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3586 type = "AOD";
3587 comment += "AOD";
3588 }
3589 }
0df6ccf2 3590 if (type!="AOD" && fFriendChainName!="") {
3591 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3592 return;
3593 }
c57f56b7 3594 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3595 else comment += " data";
3596 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3597 func.ReplaceAll(".C", "");
3598 out << "void " << func.Data() << "()" << endl;
3599 out << "{" << endl;
3600 out << comment.Data() << endl;
3601 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3602 out << " TStopwatch timer;" << endl;
3603 out << " timer.Start();" << endl << endl;
13f0c00b 3604 // Change temp directory to current one
3605 out << "// Set temporary merging directory to current one" << endl;
3606 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3607 out << "// Set temporary compilation directory to current one" << endl;
3608 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3609 // Reset existing include path
3610 out << "// Reset existing include path and add current directory first in the search" << endl;
3611 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3612 if (!fExecutableCommand.Contains("aliroot")) {
3613 out << "// load base root libraries" << endl;
3614 out << " gSystem->Load(\"libTree\");" << endl;
3615 out << " gSystem->Load(\"libGeom\");" << endl;
3616 out << " gSystem->Load(\"libVMC\");" << endl;
3617 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3618 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3619 }
d5c6455a 3620 if (fAdditionalRootLibs.Length()) {
47a4137d 3621 // in principle libtree /lib geom libvmc etc. can go into this list, too
3622 out << "// Add aditional libraries" << endl;
3623 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3624 TIter next(list);
3625 TObjString *str;
3626 while((str=(TObjString*)next())) {
3627 if (str->GetString().Contains(".so"))
3628 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3629 }
47a4137d 3630 if (list) delete list;
d5c6455a 3631 }
57377eb5 3632 out << "// Load analysis framework libraries" << endl;
652561d5 3633 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3634 if (!fPackages) {
f790bc1b 3635 if (!fExecutableCommand.Contains("aliroot")) {
3636 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3637 out << " gSystem->Load(\"libESD\");" << endl;
3638 out << " gSystem->Load(\"libAOD\");" << endl;
3639 }
4e5c5506 3640 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3641 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3642 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3643 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3644 } else {
4e5c5506 3645 TIter next(fPackages);
3646 TObject *obj;
57377eb5 3647 TString pkgname;
57377eb5 3648 while ((obj=next())) {
3649 pkgname = obj->GetName();
4478e6f1 3650 if (pkgname == "STEERBase" ||
3651 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3652 if (pkgname == "ESD" ||
3653 pkgname == "ESD.par") hasESD = kTRUE;
3654 if (pkgname == "AOD" ||
3655 pkgname == "AOD.par") hasAOD = kTRUE;
3656 if (pkgname == "ANALYSIS" ||
3657 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3658 if (pkgname == "OADB" ||
3659 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3660 if (pkgname == "ANALYSISalice" ||
3661 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3662 if (pkgname == "CORRFW" ||
3663 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3664 }
3665 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3666 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3667 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3668 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3669 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3670 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3671 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3672 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3673 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3674 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3675 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3676 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3677 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3678 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3679 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3680 out << "// Compile other par packages" << endl;
3681 next.Reset();
fcc9bb6f 3682 while ((obj=next())) {
3683 pkgname = obj->GetName();
4478e6f1 3684 if (pkgname == "STEERBase" ||
3685 pkgname == "STEERBase.par" ||
3686 pkgname == "ESD" ||
3687 pkgname == "ESD.par" ||
3688 pkgname == "AOD" ||
3689 pkgname == "AOD.par" ||
3690 pkgname == "ANALYSIS" ||
3691 pkgname == "ANALYSIS.par" ||
7ea40ac8 3692 pkgname == "OADB" ||
3693 pkgname == "OADB.par" ||
4478e6f1 3694 pkgname == "ANALYSISalice" ||
3695 pkgname == "ANALYSISalice.par" ||
3696 pkgname == "CORRFW" ||
3697 pkgname == "CORRFW.par") continue;
5b273635 3698 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3699 }
4e5c5506 3700 }
b3e07543 3701 out << "// include path" << endl;
3702 // Get the include path from the interpreter and remove entries pointing to AliRoot
3703 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3704 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3705 out << " TIter nextpath(listpaths);" << endl;
3706 out << " TObjString *pname;" << endl;
3707 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3708 out << " TString current = pname->GetName();" << endl;
3709 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3710 out << " gSystem->AddIncludePath(current);" << endl;
3711 out << " }" << endl;
3712 out << " if (listpaths) delete listpaths;" << endl;
3713 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3714 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3715 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3716 if (fAdditionalLibs.Length()) {
3717 out << "// Add aditional AliRoot libraries" << endl;
3718 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3719 TIter next(list);
3720 TObjString *str;
3721 while((str=(TObjString*)next())) {
3722 if (str->GetString().Contains(".so"))
3723 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3724 if (str->GetString().Contains(".par"))
3725 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3726 }
3727 if (list) delete list;
3728 }
3729 out << endl;
c57f56b7 3730 out << "// analysis source to be compiled at runtime (if any)" << endl;
3731 if (fAnalysisSource.Length()) {
3732 TObjArray *list = fAnalysisSource.Tokenize(" ");
3733 TIter next(list);
3734 TObjString *str;
3735 while((str=(TObjString*)next())) {
3736 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3737 }
3738 if (list) delete list;
3739 }
3740 out << endl;
b3e07543 3741// out << " printf(\"Currently load libraries:\\n\");" << endl;
3742// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3743 if (fFastReadOption) {
d3339be3 3744 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3745 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3746 out << "// fast xrootd reading enabled" << endl;
3747 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3748 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3749 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3750 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3751 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3752 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3753 }
3754 if (!IsLocalTest()) {
3755 out << "// connect to AliEn and make the chain" << endl;
3756 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3757 }
c57f56b7 3758 out << "// read the analysis manager from file" << endl;
f10e8481 3759 TString analysisFile = fExecutable;
3760 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3761 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3762 << analysisFile << "\");" << endl;
3763 out << " if (!mgr) return;" << endl;
21ca8e59 3764 if (IsLocalTest()) {
3765 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3766 out << " plugin->SetRunMode(\"test\");" << endl;
3767 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3768 out << " mgr->SetGridHandler(plugin);" << endl;
3769 out << " mgr->SetDebugLevel(10);" << endl;
3770 out << " mgr->SetNSysInfo(100);" << endl;
3771 }
c57f56b7 3772 out << " mgr->PrintStatus();" << endl;
52b6a92b 3773 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3774 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3775 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3776 } else {
9a8288ff 3777 if (TestBit(AliAnalysisGrid::kTest))
3778 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3779 else
3780 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3781 }
3782 }
21ca8e59 3783 if (!IsLocalTest()) {
242accb2 3784 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3785 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3786 } else {
3787 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3788 }
f7498086 3789 out << " timer.Stop();" << endl;
3790 out << " timer.Print();" << endl;
c57f56b7 3791 out << "}" << endl << endl;
21ca8e59 3792 if (!IsLocalTest()) {
fcc9bb6f 3793 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3794 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3795 out << "{" << endl;
3796 out << "// Create a chain using url's from xml file" << endl;
242accb2 3797 out << " TString filename;" << endl;
3798 out << " Int_t run = 0;" << endl;
c85cfc0f 3799 if (IsUseMCchain()) {
3800 out << " TString treename = \"TE\";" << endl;
3801 } else {
3802 out << " TString treename = type;" << endl;
3803 out << " treename.ToLower();" << endl;
3804 out << " treename += \"Tree\";" << endl;
3805 }
e02fee64 3806 out << " printf(\"***************************************\\n\");" << endl;
3807 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3808 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3809 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3810 out << " if (!coll) {" << endl;
3811 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3812 out << " return NULL;" << endl;
3813 out << " }" << endl;
242accb2 3814 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3815 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3816 if(fFriendChainName!="") {
3817 out << " TChain *chainFriend = new TChain(treename);" << endl;
3818 }
c57f56b7 3819 out << " coll->Reset();" << endl;
0df6ccf2 3820 out << " while (coll->Next()) {" << endl;
242accb2 3821 out << " filename = coll->GetTURL("");" << endl;
3822 out << " if (mgr) {" << endl;
3823 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3824 out << " if (nrun && nrun != run) {" << endl;
3825 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3826 out << " mgr->SetRunFromPath(nrun);" << endl;
3827 out << " run = nrun;" << endl;
3828 out << " }" << endl;
3829 out << " }" << endl;
3830 out << " chain->Add(filename);" << endl;
0df6ccf2 3831 if(fFriendChainName!="") {
3832 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3833 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3834 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3835 out << " chainFriend->Add(fileFriend.Data());" << endl;
3836 }
3837 out << " }" << endl;
c57f56b7 3838 out << " if (!chain->GetNtrees()) {" << endl;
3839 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3840 out << " return NULL;" << endl;
3841 out << " }" << endl;
0df6ccf2 3842 if(fFriendChainName!="") {
3843 out << " chain->AddFriend(chainFriend);" << endl;
3844 }
c57f56b7 3845 out << " return chain;" << endl;
fcc9bb6f 3846 out << "}" << endl << endl;
c57f56b7 3847 }
5b273635 3848 if (hasANALYSISalice) {
3849 out <<"//________________________________________________________________________________" << endl;
3850 out << "Bool_t SetupPar(const char *package) {" << endl;
3851 out << "// Compile the package and set it up." << endl;
3852 out << " TString pkgdir = package;" << endl;
3853 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3854 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3855 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3856 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3857 out << " // Check for BUILD.sh and execute" << endl;
3858 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3859 out << " printf(\"*******************************\\n\");" << endl;
3860 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3861 out << " printf(\"*******************************\\n\");" << endl;
3862 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3863 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3864 out << " gSystem->ChangeDirectory(cdir);" << endl;
3865 out << " return kFALSE;" << endl;
3866 out << " }" << endl;
3867 out << " } else {" << endl;
3868 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3869 out << " gSystem->ChangeDirectory(cdir);" << endl;
3870 out << " return kFALSE;" << endl;
3871 out << " }" << endl;
3872 out << " // Check for SETUP.C and execute" << endl;
3873 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3874 out << " printf(\"*******************************\\n\");" << endl;
3875 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3876 out << " printf(\"*******************************\\n\");" << endl;
3877 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3878 out << " } else {" << endl;
3879 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3880 out << " gSystem->ChangeDirectory(cdir);" << endl;
3881 out << " return kFALSE;" << endl;
3882 out << " }" << endl;
3883 out << " // Restore original workdir" << endl;
3884 out << " gSystem->ChangeDirectory(cdir);" << endl;
3885 out << " return kTRUE;" << endl;
3886 out << "}" << endl;
3887 }
c57f56b7 3888 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3889 }
3890 Bool_t copy = kTRUE;
d3b18c4c 3891 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3892 if (copy) {
3893 CdWork();
3894 TString workdir = gGrid->GetHomeDirectory();
3895 workdir += fGridWorkingDir;
3896 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3897 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3898// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3899 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3900 Form("alien://%s/%s", workdir.Data(),
3901 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3902 }
3903}
3904
0f389141 3905//______________________________________________________________________________
3906void AliAnalysisAlien::WriteMergingMacro()
3907{
3908// Write a macro to merge the outputs per master job.
3909 if (!fMergeViaJDL) return;
3910 if (!fOutputFiles.Length()) {
3911 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3912 return;
3913 }
3914 TString mergingMacro = fExecutable;
3915 mergingMacro.ReplaceAll(".sh","_merge.C");
dc030a24 3916 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3917 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3918 ofstream out;
3919 out.open(mergingMacro.Data(), ios::out);
3920 if (!out.good()) {
3921 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3922 return;
3923 }
5b273635 3924 Bool_t hasSTEERBase = kFALSE;
3925 Bool_t hasESD = kFALSE;
3926 Bool_t hasAOD = kFALSE;
3927 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3928 Bool_t hasOADB = kFALSE;
5b273635 3929 Bool_t hasANALYSISalice = kFALSE;
3930 Bool_t hasCORRFW = kFALSE;
0f389141 3931 TString func = mergingMacro;
3932 TString comment;
3933 func.ReplaceAll(".C", "");
f47d5cb4 3934 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3935 out << "{" << endl;
3936 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3937 out << " TStopwatch timer;" << endl;
3938 out << " timer.Start();" << endl << endl;
b3e07543 3939 // Reset existing include path
3940 out << "// Reset existing include path and add current directory first in the search" << endl;
3941 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3942 if (!fExecutableCommand.Contains("aliroot")) {
3943 out << "// load base root libraries" << endl;
3944 out << " gSystem->Load(\"libTree\");" << endl;
3945 out << " gSystem->Load(\"libGeom\");" << endl;
3946 out << " gSystem->Load(\"libVMC\");" << endl;
3947 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3948 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3949 }
0f389141 3950 if (fAdditionalRootLibs.Length()) {
3951 // in principle libtree /lib geom libvmc etc. can go into this list, too
3952 out << "// Add aditional libraries" << endl;
3953 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3954 TIter next(list);
3955 TObjString *str;
3956 while((str=(TObjString*)next())) {
3957 if (str->GetString().Contains(".so"))
3958 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3959 }
3960 if (list) delete list;
3961 }
0f389141 3962 out << "// Load analysis framework libraries" << endl;
3963 if (!fPackages) {
7c2cd90a 3964 if (!fExecutableCommand.Contains("aliroot")) {
3965 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3966 out << " gSystem->Load(\"libESD\");" << endl;
3967 out << " gSystem->Load(\"libAOD\");" << endl;
3968 }
0f389141 3969 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3970 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 3971 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3972 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3973 } else {
3974 TIter next(fPackages);
3975 TObject *obj;
3976 TString pkgname;
5b273635 3977 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3978 while ((obj=next())) {
3979 pkgname = obj->GetName();
3980 if (pkgname == "STEERBase" ||
3981 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3982 if (pkgname == "ESD" ||
3983 pkgname == "ESD.par") hasESD = kTRUE;
3984 if (pkgname == "AOD" ||
3985 pkgname == "AOD.par") hasAOD = kTRUE;
3986 if (pkgname == "ANALYSIS" ||
3987 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3988 if (pkgname == "OADB" ||
3989 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 3990 if (pkgname == "ANALYSISalice" ||
3991 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3992 if (pkgname == "CORRFW" ||
3993 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3994 }
5b273635 3995 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3996 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3997 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3998 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3999 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4000 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4001 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4002 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4003 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4004 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4005 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4006 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4007 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4008 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4009 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4010 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4011 out << "// Compile other par packages" << endl;
4012 next.Reset();
4013 while ((obj=next())) {
4014 pkgname = obj->GetName();
4015 if (pkgname == "STEERBase" ||
4016 pkgname == "STEERBase.par" ||
4017 pkgname == "ESD" ||
4018 pkgname == "ESD.par" ||
4019 pkgname == "AOD" ||
4020 pkgname == "AOD.par" ||
4021 pkgname == "ANALYSIS" ||
4022 pkgname == "ANALYSIS.par" ||
7ea40ac8 4023 pkgname == "OADB" ||
4024 pkgname == "OADB.par" ||
0f389141 4025 pkgname == "ANALYSISalice" ||
4026 pkgname == "ANALYSISalice.par" ||
4027 pkgname == "CORRFW" ||
4028 pkgname == "CORRFW.par") continue;
5b273635 4029 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4030 }
4031 }
b3e07543 4032 out << "// include path" << endl;
4033 // Get the include path from the interpreter and remove entries pointing to AliRoot
4034 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4035 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4036 out << " TIter nextpath(listpaths);" << endl;
4037 out << " TObjString *pname;" << endl;
4038 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4039 out << " TString current = pname->GetName();" << endl;
4040 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4041 out << " gSystem->AddIncludePath(current);" << endl;
4042 out << " }" << endl;
4043 out << " if (listpaths) delete listpaths;" << endl;
4044 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4045 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4046 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4047 if (fAdditionalLibs.Length()) {
4048 out << "// Add aditional AliRoot libraries" << endl;
4049 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4050 TIter next(list);
4051 TObjString *str;
4052 while((str=(TObjString*)next())) {
4053 if (str->GetString().Contains(".so"))
4054 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4055 }
4056 if (list) delete list;
4057 }
4058 out << endl;
4059 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4060 if (fAnalysisSource.Length()) {
4061 TObjArray *list = fAnalysisSource.Tokenize(" ");
4062 TIter next(list);
4063 TObjString *str;
4064 while((str=(TObjString*)next())) {
4065 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4066 }
4067 if (list) delete list;
4068 }
149d288c 4069 out << endl;
4070
0f389141 4071 if (fFastReadOption) {
4072 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4073 out << "// fast xrootd reading enabled" << endl;
4074 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4075 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4076 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4077 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4078 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4079 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4080 }
4081 // Change temp directory to current one
4082 out << "// Set temporary merging directory to current one" << endl;
4083 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4084 out << "// Set temporary compilation directory to current one" << endl;
4085 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4086 out << "// Connect to AliEn" << endl;
4087 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4088 out << " TString outputDir = dir;" << endl;
f790bc1b 4089 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 4090 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 4091 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4092 out << " TIter *iter = new TIter(list);" << endl;
4093 out << " TObjString *str;" << endl;
a2f5fc01 4094 out << " TString outputFile;" << endl;
0f389141 4095 out << " Bool_t merged = kTRUE;" << endl;
4096 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4097 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4098 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4099 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4100 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4101 out << " // Skip already merged outputs" << endl;
a2f5fc01 4102 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4103 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4104 out << " continue;" << endl;
4105 out << " }" << endl;
a2f5fc01 4106 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4107 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4108 out << " if (!merged) {" << endl;
a2f5fc01 4109 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4110 out << " return;" << endl;
0f389141 4111 out << " }" << endl;
4112 out << " }" << endl;
7c2cd90a 4113 out << " // all outputs merged, validate" << endl;
4114 out << " ofstream out;" << endl;
4115 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4116 out << " out.close();" << endl;
4117 out << " // read the analysis manager from file" << endl;
f866cba5 4118 TString analysisFile = fExecutable;
4119 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4120 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4121 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4122 << analysisFile << "\");" << endl;
4123 out << " if (!mgr) return;" << endl;
5c4250fc 4124 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4125 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4126 out << " mgr->PrintStatus();" << endl;
4127 if (AliAnalysisManager::GetAnalysisManager()) {
4128 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4129 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4130 } else {
9a8288ff 4131 if (TestBit(AliAnalysisGrid::kTest))
4132 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4133 else
4134 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4135 }
4136 }
9a8288ff 4137 out << " TTree *tree = NULL;" << endl;
4138 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4139 out << "}" << endl << endl;
5b273635 4140 if (hasANALYSISalice) {
4141 out <<"//________________________________________________________________________________" << endl;
4142 out << "Bool_t SetupPar(const char *package) {" << endl;
4143 out << "// Compile the package and set it up." << endl;
4144 out << " TString pkgdir = package;" << endl;
4145 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4146 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4147 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4148 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4149 out << " // Check for BUILD.sh and execute" << endl;
4150 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4151 out << " printf(\"*******************************\\n\");" << endl;
4152 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4153 out << " printf(\"*******************************\\n\");" << endl;
4154 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4155 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4156 out << " gSystem->ChangeDirectory(cdir);" << endl;
4157 out << " return kFALSE;" << endl;
4158 out << " }" << endl;
4159 out << " } else {" << endl;
4160 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4161 out << " gSystem->ChangeDirectory(cdir);" << endl;
4162 out << " return kFALSE;" << endl;
4163 out << " }" << endl;
4164 out << " // Check for SETUP.C and execute" << endl;
4165 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4166 out << " printf(\"*******************************\\n\");" << endl;
4167 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4168 out << " printf(\"*******************************\\n\");" << endl;
4169 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4170 out << " } else {" << endl;
4171 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4172 out << " gSystem->ChangeDirectory(cdir);" << endl;
4173 out << " return kFALSE;" << endl;
4174 out << " }" << endl;
4175 out << " // Restore original workdir" << endl;
4176 out << " gSystem->ChangeDirectory(cdir);" << endl;
4177 out << " return kTRUE;" << endl;
4178 out << "}" << endl;
4179 }
0f389141 4180 }
4181 Bool_t copy = kTRUE;
d3b18c4c 4182 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4183 if (copy) {
4184 CdWork();
4185 TString workdir = gGrid->GetHomeDirectory();
4186 workdir += fGridWorkingDir;
4187 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4188 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4189// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4190 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4191 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4192 }
4193}
4194
4195//______________________________________________________________________________
4196Bool_t AliAnalysisAlien::SetupPar(const char *package)
4197{
205b201f 4198// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4199// Note that for loading the compiled library. The current directory should have precedence in
4200// LD_LIBRARY_PATH
4201 TString pkgdir = package;
4202 pkgdir.ReplaceAll(".par","");
b50a41c3 4203 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4204 TString cdir = gSystem->WorkingDirectory();
4205 gSystem->ChangeDirectory(pkgdir);
4206 // Check for BUILD.sh and execute
4207 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4208 printf("**************************************************\n");
4209 printf("*** Building PAR archive %s\n", package);
4210 printf("**************************************************\n");
4211 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4212 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4213 gSystem->ChangeDirectory(cdir);
4214 return kFALSE;
4215 }
4216 } else {
4217 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4218 gSystem->ChangeDirectory(cdir);
4219 return kFALSE;
4220 }
4221 // Check for SETUP.C and execute
4222 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4223 printf("**************************************************\n");
4224 printf("*** Setup PAR archive %s\n", package);
4225 printf("**************************************************\n");
4226 gROOT->Macro("PROOF-INF/SETUP.C");
4227 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4228 } else {
4229 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4230 gSystem->ChangeDirectory(cdir);
4231 return kFALSE;
4232 }
4233 // Restore original workdir
4234 gSystem->ChangeDirectory(cdir);
4235 return kTRUE;
4236}
4237
c57f56b7 4238//______________________________________________________________________________
4239void AliAnalysisAlien::WriteExecutable()
4240{
4241// Generate the alien executable script.
4242 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4243 ofstream out;
4244 out.open(fExecutable.Data(), ios::out);
4245 if (out.bad()) {
5513444a 4246 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4247 return;
4248 }
4249 out << "#!/bin/bash" << endl;
2a4d5166 4250 // Make sure we can properly compile par files
4251 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4252 out << "echo \"=========================================\"" << endl;
4253 out << "echo \"############## PATH : ##############\"" << endl;
4254 out << "echo $PATH" << endl;
4255 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4256 out << "echo $LD_LIBRARY_PATH" << endl;
4257 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4258 out << "echo $ROOTSYS" << endl;
4259 out << "echo \"############## which root : ##############\"" << endl;
4260 out << "which root" << endl;
4261 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4262 out << "echo $ALICE_ROOT" << endl;
4263 out << "echo \"############## which aliroot : ##############\"" << endl;
4264 out << "which aliroot" << endl;
9c5ddadc 4265 out << "echo \"############## system limits : ##############\"" << endl;
4266 out << "ulimit -a" << endl;
4267 out << "echo \"############## memory : ##############\"" << endl;
4268 out << "free -m" << endl;
c57f56b7 4269 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4270 out << fExecutableCommand << " ";
631c0b05 4271 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 4272 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
4273 out << "echo \"############## memory after: ##############\"" << endl;
4274 out << "free -m" << endl;
c57f56b7 4275 }
4276 Bool_t copy = kTRUE;
d3b18c4c 4277 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4278 if (copy) {
4279 CdWork();
4280 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4281 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4282 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4283 workdir += fGridWorkingDir;
4284 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4285 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4286 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4287// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4288 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4289 executable.Data())) Fatal("","Terminating");
c57f56b7 4290 }
4291}
4292
0f389141 4293//______________________________________________________________________________
4294void AliAnalysisAlien::WriteMergeExecutable()
4295{
4296// Generate the alien executable script for the merging job.
4297 if (!fMergeViaJDL) return;
4298 TString mergeExec = fExecutable;
4299 mergeExec.ReplaceAll(".sh", "_merge.sh");
4300 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4301 ofstream out;
4302 out.open(mergeExec.Data(), ios::out);
4303 if (out.bad()) {
4304 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4305 return;
4306 }
4307 out << "#!/bin/bash" << endl;
2a4d5166 4308 // Make sure we can properly compile par files
4309 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4310 out << "echo \"=========================================\"" << endl;
4311 out << "echo \"############## PATH : ##############\"" << endl;
4312 out << "echo $PATH" << endl;
4313 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4314 out << "echo $LD_LIBRARY_PATH" << endl;
4315 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4316 out << "echo $ROOTSYS" << endl;
4317 out << "echo \"############## which root : ##############\"" << endl;
4318 out << "which root" << endl;
4319 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4320 out << "echo $ALICE_ROOT" << endl;
4321 out << "echo \"############## which aliroot : ##############\"" << endl;
4322 out << "which aliroot" << endl;
4323 out << "echo \"############## system limits : ##############\"" << endl;
4324 out << "ulimit -a" << endl;
4325 out << "echo \"############## memory : ##############\"" << endl;
4326 out << "free -m" << endl;
4327 out << "echo \"=========================================\"" << endl << endl;
0f389141 4328 TString mergeMacro = fExecutable;
4329 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4330 if (IsOneStageMerging())
4331 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4332 else
f47d5cb4 4333 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4334 out << fExecutableCommand << " " << "$ARG" << endl;
4335 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4336 out << "echo \"############## memory after: ##############\"" << endl;
4337 out << "free -m" << endl;
0f389141 4338 }
4339 Bool_t copy = kTRUE;
d3b18c4c 4340 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4341 if (copy) {
4342 CdWork();
4343 TString workdir = gGrid->GetHomeDirectory();
4344 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4345 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4346 workdir += fGridWorkingDir;
4347 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4348 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4349 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4350// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4351 if (!copyLocal2Alien("WriteMergeExecutable",
4352 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4353 }
4354}
4355
c57f56b7 4356//______________________________________________________________________________
5513444a 4357void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4358{
4359// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4360// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4361// Next lines: full_path_to_dataset XXX (XXX is a string)
4362// To submit, one has to: submit jdl XXX for all lines
4363 ofstream out;
4364 out.open(filename, ios::out);
4365 if (out.bad()) {
4366 Error("WriteProductionFile", "Bad file name: %s", filename);
4367 return;
4368 }
d3b18c4c 4369 TString workdir;
4370 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4371 workdir = gGrid->GetHomeDirectory();
5513444a 4372 workdir += fGridWorkingDir;
f5e8c702 4373 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4374 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4375 out << locjdl << " " << njobspermaster << endl;
5513444a 4376 Int_t nmasterjobs = fInputFiles->GetEntries();
4377 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4378 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4379 runOutDir.ReplaceAll(".xml", "");
4380 if (fOutputToRunNo)
4381 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4382 else
4383 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4384 }
d3b18c4c 4385 if (gGrid) {
4386 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4387 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4388// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4389 if (!copyLocal2Alien("WriteProductionFile", filename,
4390 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4391 }
5513444a 4392}
4393
4394//______________________________________________________________________________
0f389141 4395void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4396{
4397// Generate the alien validation script.
4398 // Generate the validation script
4399 TObjString *os;
d3b18c4c 4400 if (fValidationScript.IsNull()) {
4401 fValidationScript = fExecutable;
4402 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4403 }
4404 TString validationScript = fValidationScript;
4405 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4406 if (!Connect()) {
4407 Error("WriteValidationScript", "Alien connection required");
4408 return;
4409 }
d3b18c4c 4410 if (!fTerminateFiles.IsNull()) {
4411 fTerminateFiles.Strip();
4412 fTerminateFiles.ReplaceAll(" ",",");
4413 }
a2f5fc01 4414 TString outStream = "";
4415 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4416 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4417 ofstream out;
0d5d317c 4418 out.open(validationScript, ios::out);
c57f56b7 4419 out << "#!/bin/bash" << endl;
4420 out << "##################################################" << endl;
4421 out << "validateout=`dirname $0`" << endl;
4422 out << "validatetime=`date`" << endl;
4423 out << "validated=\"0\";" << endl;
4424 out << "error=0" << endl;
4425 out << "if [ -z $validateout ]" << endl;
4426 out << "then" << endl;
4427 out << " validateout=\".\"" << endl;
4428 out << "fi" << endl << endl;
4429 out << "cd $validateout;" << endl;
4430 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4431 out << "echo \"*******************************************************\"" << outStream << endl;
4432 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4433 out << "" << endl;
a2f5fc01 4434 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4435 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4436 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4437 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4438 out << "ls -la ./" << outStream << endl;
4439 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4440 out << "##################################################" << endl;
ebec370a 4441 out << "" << endl;
4442
4443 out << "if [ ! -f stderr ] ; then" << endl;
4444 out << " error=1" << endl;
a2f5fc01 4445 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4446 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4447 out << "fi" << endl;
4448
b34c9f51 4449 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4450 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4451 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4452 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4453 out << "" << endl;
4454
ebec370a 4455 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4456 out << " error=1" << endl;
a2f5fc01 4457 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4458 out << " echo \"$parArch\" " << outStream << endl;
4459 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4460 out << "fi" << endl;
4461
4462 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4463 out << " error=1" << endl;
a2f5fc01 4464 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4465 out << " echo \"$segViol\" " << outStream << endl;
4466 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4467 out << "fi" << endl;
4468
4469 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4470 out << " error=1" << endl;
a2f5fc01 4471 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4472 out << " echo \"$segFault\" " << outStream << endl;
4473 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4474 out << "fi" << endl;
4475
b34c9f51 4476 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4477 out << " error=1" << endl;
a2f5fc01 4478 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4479 out << " echo \"$glibcErr\" " << outStream << endl;
4480 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4481 out << "fi" << endl;
4482
ebec370a 4483 // Part dedicated to the specific analyses running into the train
4484
d3b18c4c 4485 TString outputFiles = fOutputFiles;
4486 if (merge && !fTerminateFiles.IsNull()) {
4487 outputFiles += ",";
4488 outputFiles += fTerminateFiles;
4489 }
4490 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4491 TIter next1(arr);
a2f5fc01 4492 TString outputFile;
bbd88db8 4493 while (!merge && (os=(TObjString*)next1())) {
4494 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4495 outputFile = os->GetString();
4496 Int_t index = outputFile.Index("@");
4497 if (index > 0) outputFile.Remove(index);
bbd88db8 4498 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4499 if (outputFile.Contains("*")) continue;
4500 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4501 out << " error=1" << endl;
7c2cd90a 4502 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4503 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4504 out << "fi" << endl;
4505 }
4506 delete arr;
7c2cd90a 4507 out << "if ! [ -f outputs_valid ] ; then" << endl;
4508 out << " error=1" << endl;
4509 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4510 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4511 out << "fi" << endl;
923e2ca5 4512
c57f56b7 4513 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4514 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4515 if (!IsKeepLogs()) {
4516 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4517 outStream = "";
149d288c 4518 out << " rm -f std*" << endl;
4519 }
c57f56b7 4520 out << "fi" << endl;
4521
a2f5fc01 4522 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4523 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4524 out << "cd -" << endl;
4525 out << "exit $error" << endl;
4526 }
4527 Bool_t copy = kTRUE;
d3b18c4c 4528 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4529 if (copy) {
4530 CdWork();
4531 TString workdir = gGrid->GetHomeDirectory();
4532 workdir += fGridWorkingDir;
d3b18c4c 4533 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4534 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4535// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4536 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4537 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4538 }
4539}