]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Bug fixes. Trigger classes selection is added.
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
51ClassImp(AliAnalysisAlien)
6c395669 52#if 0
53;
54#endif
c57f56b7 55
6c395669 56namespace {
57 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
58 {
59 TString sl(Form("file:%s", loc));
60 TString sr(Form("alien://%s", rem));
61 Bool_t ret = TFile::Cp(sl, sr);
62 if (!ret) {
63 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
64 }
65 return ret;
66 }
67}
68
c57f56b7 69//______________________________________________________________________________
70AliAnalysisAlien::AliAnalysisAlien()
71 :AliAnalysisGrid(),
72 fGridJDL(NULL),
0f389141 73 fMergingJDL(NULL),
c57f56b7 74 fPrice(0),
75 fTTL(0),
76 fSplitMaxInputFileNumber(0),
77 fMaxInitFailed(0),
78 fMasterResubmitThreshold(0),
bb885a9e 79 fNtestFiles(0),
319593fb 80 fNrunsPerMaster(0),
16a4353c 81 fMaxMergeFiles(0),
b3e07543 82 fMaxMergeStages(0),
d2a409b2 83 fNsubmitted(0),
a3e84053 84 fProductionMode(0),
cd11251e 85 fOutputToRunNo(0),
0f389141 86 fMergeViaJDL(0),
87 fFastReadOption(0),
e1c22e21 88 fOverwriteMode(1),
149d288c 89 fNreplicas(2),
3bdcb562 90 fNproofWorkers(0),
27734f0e 91 fNproofWorkersPerSlave(0),
3bdcb562 92 fProofReset(0),
c57f56b7 93 fRunNumbers(),
94 fExecutable(),
0a1c1f7f 95 fExecutableCommand(),
c57f56b7 96 fArguments(),
631c0b05 97 fExecutableArgs(),
c57f56b7 98 fAnalysisMacro(),
99 fAnalysisSource(),
d3b18c4c 100 fValidationScript(),
d5c6455a 101 fAdditionalRootLibs(),
c57f56b7 102 fAdditionalLibs(),
103 fSplitMode(),
104 fAPIVersion(),
105 fROOTVersion(),
106 fAliROOTVersion(),
648174cf 107 fExternalPackages(),
c57f56b7 108 fUser(),
109 fGridWorkingDir(),
110 fGridDataDir(),
111 fDataPattern(),
112 fGridOutputDir(),
113 fOutputArchive(),
114 fOutputFiles(),
115 fInputFormat(),
e7c71df0 116 fDatasetName(),
c57f56b7 117 fJDLName(),
d3b18c4c 118 fTerminateFiles(),
bb885a9e 119 fMergeExcludes(),
f965131e 120 fIncludePath(),
bb885a9e 121 fCloseSE(),
0df6ccf2 122 fFriendChainName(),
c6cb3634 123 fJobTag(),
648174cf 124 fOutputSingle(),
5fce53f4 125 fRunPrefix(),
3bdcb562 126 fProofCluster(),
127 fProofDataSet(),
128 fFileForTestMode(),
129 fRootVersionForProof(),
130 fAliRootMode(),
f47d5cb4 131 fMergeDirName(),
4e5c5506 132 fInputFiles(0),
28ce6c9c 133 fPackages(0),
77f34eae 134 fModules(0),
28ce6c9c 135 fProofParam()
c57f56b7 136{
137// Dummy ctor.
138 SetDefaults();
139}
140
141//______________________________________________________________________________
142AliAnalysisAlien::AliAnalysisAlien(const char *name)
143 :AliAnalysisGrid(name),
144 fGridJDL(NULL),
0f389141 145 fMergingJDL(NULL),
c57f56b7 146 fPrice(0),
147 fTTL(0),
148 fSplitMaxInputFileNumber(0),
149 fMaxInitFailed(0),
150 fMasterResubmitThreshold(0),
bb885a9e 151 fNtestFiles(0),
319593fb 152 fNrunsPerMaster(0),
16a4353c 153 fMaxMergeFiles(0),
b3e07543 154 fMaxMergeStages(0),
d2a409b2 155 fNsubmitted(0),
a3e84053 156 fProductionMode(0),
cd11251e 157 fOutputToRunNo(0),
0f389141 158 fMergeViaJDL(0),
159 fFastReadOption(0),
e1c22e21 160 fOverwriteMode(1),
149d288c 161 fNreplicas(2),
3bdcb562 162 fNproofWorkers(0),
27734f0e 163 fNproofWorkersPerSlave(0),
3bdcb562 164 fProofReset(0),
c57f56b7 165 fRunNumbers(),
166 fExecutable(),
0a1c1f7f 167 fExecutableCommand(),
c57f56b7 168 fArguments(),
631c0b05 169 fExecutableArgs(),
c57f56b7 170 fAnalysisMacro(),
171 fAnalysisSource(),
d3b18c4c 172 fValidationScript(),
d5c6455a 173 fAdditionalRootLibs(),
c57f56b7 174 fAdditionalLibs(),
175 fSplitMode(),
176 fAPIVersion(),
177 fROOTVersion(),
178 fAliROOTVersion(),
648174cf 179 fExternalPackages(),
c57f56b7 180 fUser(),
181 fGridWorkingDir(),
182 fGridDataDir(),
183 fDataPattern(),
184 fGridOutputDir(),
185 fOutputArchive(),
186 fOutputFiles(),
187 fInputFormat(),
e7c71df0 188 fDatasetName(),
c57f56b7 189 fJDLName(),
d3b18c4c 190 fTerminateFiles(),
bb885a9e 191 fMergeExcludes(),
f965131e 192 fIncludePath(),
bb885a9e 193 fCloseSE(),
0df6ccf2 194 fFriendChainName(),
c6cb3634 195 fJobTag(),
648174cf 196 fOutputSingle(),
5fce53f4 197 fRunPrefix(),
3bdcb562 198 fProofCluster(),
199 fProofDataSet(),
200 fFileForTestMode(),
201 fRootVersionForProof(),
202 fAliRootMode(),
f47d5cb4 203 fMergeDirName(),
4e5c5506 204 fInputFiles(0),
28ce6c9c 205 fPackages(0),
77f34eae 206 fModules(0),
28ce6c9c 207 fProofParam()
c57f56b7 208{
209// Default ctor.
210 SetDefaults();
211}
212
213//______________________________________________________________________________
214AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
215 :AliAnalysisGrid(other),
216 fGridJDL(NULL),
0f389141 217 fMergingJDL(NULL),
c57f56b7 218 fPrice(other.fPrice),
219 fTTL(other.fTTL),
220 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
221 fMaxInitFailed(other.fMaxInitFailed),
222 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 223 fNtestFiles(other.fNtestFiles),
319593fb 224 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 225 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 226 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 227 fNsubmitted(other.fNsubmitted),
a3e84053 228 fProductionMode(other.fProductionMode),
cd11251e 229 fOutputToRunNo(other.fOutputToRunNo),
0f389141 230 fMergeViaJDL(other.fMergeViaJDL),
231 fFastReadOption(other.fFastReadOption),
232 fOverwriteMode(other.fOverwriteMode),
149d288c 233 fNreplicas(other.fNreplicas),
3bdcb562 234 fNproofWorkers(other.fNproofWorkers),
27734f0e 235 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 236 fProofReset(other.fProofReset),
c57f56b7 237 fRunNumbers(other.fRunNumbers),
238 fExecutable(other.fExecutable),
0a1c1f7f 239 fExecutableCommand(other.fExecutableCommand),
c57f56b7 240 fArguments(other.fArguments),
631c0b05 241 fExecutableArgs(other.fExecutableArgs),
c57f56b7 242 fAnalysisMacro(other.fAnalysisMacro),
243 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 244 fValidationScript(other.fValidationScript),
d5c6455a 245 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 246 fAdditionalLibs(other.fAdditionalLibs),
247 fSplitMode(other.fSplitMode),
248 fAPIVersion(other.fAPIVersion),
249 fROOTVersion(other.fROOTVersion),
250 fAliROOTVersion(other.fAliROOTVersion),
648174cf 251 fExternalPackages(other.fExternalPackages),
c57f56b7 252 fUser(other.fUser),
253 fGridWorkingDir(other.fGridWorkingDir),
254 fGridDataDir(other.fGridDataDir),
255 fDataPattern(other.fDataPattern),
256 fGridOutputDir(other.fGridOutputDir),
257 fOutputArchive(other.fOutputArchive),
258 fOutputFiles(other.fOutputFiles),
259 fInputFormat(other.fInputFormat),
e7c71df0 260 fDatasetName(other.fDatasetName),
c57f56b7 261 fJDLName(other.fJDLName),
d3b18c4c 262 fTerminateFiles(other.fTerminateFiles),
bb885a9e 263 fMergeExcludes(other.fMergeExcludes),
f965131e 264 fIncludePath(other.fIncludePath),
bb885a9e 265 fCloseSE(other.fCloseSE),
0df6ccf2 266 fFriendChainName(other.fFriendChainName),
c6cb3634 267 fJobTag(other.fJobTag),
648174cf 268 fOutputSingle(other.fOutputSingle),
5fce53f4 269 fRunPrefix(other.fRunPrefix),
3bdcb562 270 fProofCluster(other.fProofCluster),
271 fProofDataSet(other.fProofDataSet),
272 fFileForTestMode(other.fFileForTestMode),
273 fRootVersionForProof(other.fRootVersionForProof),
274 fAliRootMode(other.fAliRootMode),
f47d5cb4 275 fMergeDirName(other.fMergeDirName),
4e5c5506 276 fInputFiles(0),
28ce6c9c 277 fPackages(0),
77f34eae 278 fModules(0),
28ce6c9c 279 fProofParam()
c57f56b7 280{
281// Copy ctor.
282 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 283 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 284 fRunRange[0] = other.fRunRange[0];
285 fRunRange[1] = other.fRunRange[1];
c57f56b7 286 if (other.fInputFiles) {
287 fInputFiles = new TObjArray();
288 TIter next(other.fInputFiles);
289 TObject *obj;
290 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
291 fInputFiles->SetOwner();
292 }
4e5c5506 293 if (other.fPackages) {
294 fPackages = new TObjArray();
295 TIter next(other.fPackages);
296 TObject *obj;
297 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
298 fPackages->SetOwner();
299 }
77f34eae 300 if (other.fModules) {
301 fModules = new TObjArray();
302 fModules->SetOwner();
303 TIter next(other.fModules);
304 AliAnalysisTaskCfg *mod, *crt;
305 while ((crt=(AliAnalysisTaskCfg*)next())) {
306 mod = new AliAnalysisTaskCfg(*crt);
307 fModules->Add(mod);
308 }
309 }
c57f56b7 310}
311
312//______________________________________________________________________________
313AliAnalysisAlien::~AliAnalysisAlien()
314{
315// Destructor.
77f34eae 316 delete fGridJDL;
317 delete fMergingJDL;
318 delete fInputFiles;
319 delete fPackages;
320 delete fModules;
28ce6c9c 321 fProofParam.DeleteAll();
c57f56b7 322}
323
324//______________________________________________________________________________
325AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
326{
327// Assignment.
328 if (this != &other) {
329 AliAnalysisGrid::operator=(other);
330 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 331 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 332 fPrice = other.fPrice;
333 fTTL = other.fTTL;
334 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
335 fMaxInitFailed = other.fMaxInitFailed;
336 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 337 fNtestFiles = other.fNtestFiles;
a3e84053 338 fNrunsPerMaster = other.fNrunsPerMaster;
339 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 340 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 341 fNsubmitted = other.fNsubmitted;
342 fProductionMode = other.fProductionMode;
cd11251e 343 fOutputToRunNo = other.fOutputToRunNo;
0f389141 344 fMergeViaJDL = other.fMergeViaJDL;
345 fFastReadOption = other.fFastReadOption;
346 fOverwriteMode = other.fOverwriteMode;
149d288c 347 fNreplicas = other.fNreplicas;
3bdcb562 348 fNproofWorkers = other.fNproofWorkers;
27734f0e 349 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 350 fProofReset = other.fProofReset;
c57f56b7 351 fRunNumbers = other.fRunNumbers;
352 fExecutable = other.fExecutable;
0a1c1f7f 353 fExecutableCommand = other.fExecutableCommand;
c57f56b7 354 fArguments = other.fArguments;
631c0b05 355 fExecutableArgs = other.fExecutableArgs;
c57f56b7 356 fAnalysisMacro = other.fAnalysisMacro;
357 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 358 fValidationScript = other.fValidationScript;
d5c6455a 359 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 360 fAdditionalLibs = other.fAdditionalLibs;
361 fSplitMode = other.fSplitMode;
362 fAPIVersion = other.fAPIVersion;
363 fROOTVersion = other.fROOTVersion;
364 fAliROOTVersion = other.fAliROOTVersion;
648174cf 365 fExternalPackages = other.fExternalPackages;
c57f56b7 366 fUser = other.fUser;
367 fGridWorkingDir = other.fGridWorkingDir;
368 fGridDataDir = other.fGridDataDir;
369 fDataPattern = other.fDataPattern;
370 fGridOutputDir = other.fGridOutputDir;
371 fOutputArchive = other.fOutputArchive;
372 fOutputFiles = other.fOutputFiles;
373 fInputFormat = other.fInputFormat;
e7c71df0 374 fDatasetName = other.fDatasetName;
c57f56b7 375 fJDLName = other.fJDLName;
d3b18c4c 376 fTerminateFiles = other.fTerminateFiles;
bb885a9e 377 fMergeExcludes = other.fMergeExcludes;
f965131e 378 fIncludePath = other.fIncludePath;
bb885a9e 379 fCloseSE = other.fCloseSE;
0df6ccf2 380 fFriendChainName = other.fFriendChainName;
c6cb3634 381 fJobTag = other.fJobTag;
648174cf 382 fOutputSingle = other.fOutputSingle;
5fce53f4 383 fRunPrefix = other.fRunPrefix;
3bdcb562 384 fProofCluster = other.fProofCluster;
385 fProofDataSet = other.fProofDataSet;
386 fFileForTestMode = other.fFileForTestMode;
387 fRootVersionForProof = other.fRootVersionForProof;
388 fAliRootMode = other.fAliRootMode;
f47d5cb4 389 fMergeDirName = other.fMergeDirName;
c57f56b7 390 if (other.fInputFiles) {
391 fInputFiles = new TObjArray();
392 TIter next(other.fInputFiles);
393 TObject *obj;
394 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
395 fInputFiles->SetOwner();
396 }
4e5c5506 397 if (other.fPackages) {
398 fPackages = new TObjArray();
399 TIter next(other.fPackages);
400 TObject *obj;
401 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
402 fPackages->SetOwner();
403 }
77f34eae 404 if (other.fModules) {
405 fModules = new TObjArray();
406 fModules->SetOwner();
407 TIter next(other.fModules);
408 AliAnalysisTaskCfg *mod, *crt;
409 while ((crt=(AliAnalysisTaskCfg*)next())) {
410 mod = new AliAnalysisTaskCfg(*crt);
411 fModules->Add(mod);
412 }
413 }
c57f56b7 414 }
415 return *this;
416}
417
77f34eae 418//______________________________________________________________________________
419void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
420{
421// Adding a module. Checks if already existing. Becomes owned by this.
422 if (!module) return;
423 if (GetModule(module->GetName())) {
424 Error("AddModule", "A module having the same name %s already added", module->GetName());
425 return;
426 }
427 if (!fModules) {
428 fModules = new TObjArray();
429 fModules->SetOwner();
430 }
431 fModules->Add(module);
432}
433
434//______________________________________________________________________________
435void AliAnalysisAlien::AddModules(TObjArray *list)
436{
437// Adding a list of modules. Checks if already existing. Becomes owned by this.
438 TIter next(list);
439 AliAnalysisTaskCfg *module;
440 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
441}
442
443//______________________________________________________________________________
444Bool_t AliAnalysisAlien::CheckDependencies()
445{
446// Check if all dependencies are satisfied. Reorder modules if needed.
447 Int_t nmodules = GetNmodules();
448 if (!nmodules) {
449 Warning("CheckDependencies", "No modules added yet to check their dependencies");
450 return kTRUE;
451 }
452 AliAnalysisTaskCfg *mod = 0;
453 AliAnalysisTaskCfg *dep = 0;
454 TString depname;
455 Int_t i, j, k;
456 for (i=0; i<nmodules; i++) {
457 mod = (AliAnalysisTaskCfg*) fModules->At(i);
458 Int_t ndeps = mod->GetNdeps();
459 Int_t istart = i;
460 for (j=0; j<ndeps; j++) {
461 depname = mod->GetDependency(j);
462 dep = GetModule(depname);
463 if (!dep) {
464 Error("CheckDependencies","Dependency %s not added for module %s",
465 depname.Data(), mod->GetName());
466 return kFALSE;
467 }
468 if (dep->NeedsDependency(mod->GetName())) {
469 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
470 mod->GetName(), dep->GetName());
471 return kFALSE;
472 }
473 Int_t idep = fModules->IndexOf(dep);
474 // The dependency task must come first
475 if (idep>i) {
476 // Remove at idep and move all objects below up one slot
477 // down to index i included.
478 fModules->RemoveAt(idep);
3e40fd4c 479 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 480 fModules->AddAt(dep, i++);
481 }
482 //Redo from istart if dependencies were inserted
483 if (i>istart) i=istart-1;
484 }
485 }
486 return kTRUE;
487}
488
3e40fd4c 489//______________________________________________________________________________
490AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
491{
492// Create the analysis manager and optionally execute the macro in filename.
493 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
494 if (mgr) return mgr;
495 mgr = new AliAnalysisManager(name);
496 mgr->SetGridHandler((AliAnalysisGrid*)this);
497 if (strlen(filename)) {
498 TString line = gSystem->ExpandPathName(filename);
499 line.Prepend(".x ");
500 gROOT->ProcessLine(line.Data());
501 }
502 return mgr;
503}
504
77f34eae 505//______________________________________________________________________________
506Int_t AliAnalysisAlien::GetNmodules() const
507{
508// Get number of modules.
509 if (!fModules) return 0;
510 return fModules->GetEntries();
511}
512
513//______________________________________________________________________________
514AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
515{
516// Get a module by name.
517 if (!fModules) return 0;
518 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
519}
520
521//______________________________________________________________________________
522Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
523{
524// Load a given module.
525 if (mod->IsLoaded()) return kTRUE;
526 Int_t ndeps = mod->GetNdeps();
527 TString depname;
528 for (Int_t j=0; j<ndeps; j++) {
529 depname = mod->GetDependency(j);
530 AliAnalysisTaskCfg *dep = GetModule(depname);
531 if (!dep) {
532 Error("LoadModule","Dependency %s not existing for module %s",
533 depname.Data(), mod->GetName());
534 return kFALSE;
535 }
536 if (!LoadModule(dep)) {
537 Error("LoadModule","Dependency %s for module %s could not be loaded",
538 depname.Data(), mod->GetName());
539 return kFALSE;
540 }
541 }
542 // Load libraries for the module
543 if (!mod->CheckLoadLibraries()) {
544 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
545 return kFALSE;
546 }
547 // Execute the macro
548 if (mod->ExecuteMacro()<0) {
549 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
550 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
551 return kFALSE;
552 }
553 // Configure dependencies
554 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
555 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
556 mod->GetConfigMacro()->GetTitle(), mod->GetName());
557 return kFALSE;
558 }
5e8c7c22 559 // Adjust extra libraries
560 Int_t nlibs = mod->GetNlibs();
561 TString lib;
562 for (Int_t i=0; i<nlibs; i++) {
563 lib = mod->GetLibrary(i);
564 if (fAdditionalLibs.Contains(lib)) continue;
565 lib = Form("lib%s.so", lib.Data());
566 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
567 fAdditionalLibs += lib;
568 }
77f34eae 569 return kTRUE;
570}
571
572//______________________________________________________________________________
21ca8e59 573Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 574{
575// Generate test macros for a single module or for the full train.
5e8c7c22 576 fAdditionalLibs = "";
77f34eae 577 if (strlen(modname)) {
21ca8e59 578 if (!CheckDependencies()) return kFALSE;
579 AliAnalysisTaskCfg *mod = GetModule(modname);
580 if (!mod) {
581 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
582 return kFALSE;
583 }
584 if (!LoadModule(mod)) return kFALSE;
585 } else if (!LoadModules()) return kFALSE;
586 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
587 if (!mgr->InitAnalysis()) return kFALSE;
588 mgr->PrintStatus();
589 SetLocalTest(kTRUE);
590 Int_t productionMode = fProductionMode;
591 SetProductionMode();
592 TString macro = fAnalysisMacro;
593 TString executable = fExecutable;
594 TString validation = fValidationScript;
595 TString execCommand = fExecutableCommand;
596 SetAnalysisMacro(Form("%s.C", name));
597 SetExecutable(Form("%s.sh", name));
598 SetExecutableCommand("aliroot -b -q ");
599 SetValidationScript(Form("%s_validation.sh", name));
600 WriteAnalysisFile();
601 WriteAnalysisMacro();
602 WriteExecutable();
603 WriteValidationScript();
604 SetLocalTest(kFALSE);
605 SetProductionMode(productionMode);
606 fAnalysisMacro = macro;
607 fExecutable = executable;
608 fExecutableCommand = execCommand;
609 fValidationScript = validation;
77f34eae 610 return kTRUE;
611}
612
613//______________________________________________________________________________
614Bool_t AliAnalysisAlien::LoadModules()
615{
616// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 617 fAdditionalLibs = "";
21ca8e59 618 Int_t nmodules = GetNmodules();
619 if (!nmodules) {
620 Warning("LoadModules", "No module to be loaded");
621 return kTRUE;
622 }
3e40fd4c 623 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
624 if (!mgr) {
625 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
626 return kFALSE;
627 }
77f34eae 628 if (!CheckDependencies()) return kFALSE;
21ca8e59 629 nmodules = GetNmodules();
77f34eae 630 AliAnalysisTaskCfg *mod;
21ca8e59 631 for (Int_t imod=0; imod<nmodules; imod++) {
632 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 633 if (!LoadModule(mod)) return kFALSE;
634 }
77f34eae 635 return kTRUE;
636}
637
74013535 638//______________________________________________________________________________
639void AliAnalysisAlien::SetRunPrefix(const char *prefix)
640{
641// Set the run number format. Can be a prefix or a format like "%09d"
642 fRunPrefix = prefix;
643 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
644}
645
f965131e 646//______________________________________________________________________________
647void AliAnalysisAlien::AddIncludePath(const char *path)
648{
649// Add include path in the remote analysis macro.
650 TString p(path);
651 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
652 else fIncludePath += Form("-I%s ", path);
653}
654
c57f56b7 655//______________________________________________________________________________
656void AliAnalysisAlien::AddRunNumber(Int_t run)
657{
658// Add a run number to the list of runs to be processed.
659 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 660 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 661}
662
98690895 663//______________________________________________________________________________
664void AliAnalysisAlien::AddRunList(const char* runList)
665{
666// Add several runs into the list of runs; they are expected to be separated by a blank character.
667 TString sList = runList;
668 TObjArray *list = sList.Tokenize(" ");
669 Int_t n = list->GetEntries();
670 for (Int_t i = 0; i < n; i++) {
671 TObjString *os = (TObjString*)list->At(i);
672 AddRunNumber(os->GetString().Atoi());
673 }
674 delete list;
675}
676
ee75cfc3 677//______________________________________________________________________________
678void AliAnalysisAlien::AddRunNumber(const char* run)
679{
680// Add a run number to the list of runs to be processed.
7aa5d539 681 TString runs = run;
682 TObjString *os;
683 TObjArray *arr = runs.Tokenize(" ");
684 TIter next(arr);
685 TString prefix;
686 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
687 while ((os=(TObjString*)next())){
688 if (fRunNumbers.Length()) fRunNumbers += " ";
689 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
690 }
691 delete arr;
ee75cfc3 692}
693
c57f56b7 694//______________________________________________________________________________
695void AliAnalysisAlien::AddDataFile(const char *lfn)
696{
697// Adds a data file to the input to be analysed. The file should be a valid LFN
698// or point to an existing file in the alien workdir.
699 if (!fInputFiles) fInputFiles = new TObjArray();
700 fInputFiles->Add(new TObjString(lfn));
701}
648174cf 702
703//______________________________________________________________________________
704void AliAnalysisAlien::AddExternalPackage(const char *package)
705{
706// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
707 if (fExternalPackages) fExternalPackages += " ";
708 fExternalPackages += package;
709}
710
c57f56b7 711//______________________________________________________________________________
712Bool_t AliAnalysisAlien::Connect()
713{
714// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
715 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 716 if (fProductionMode) return kTRUE;
c57f56b7 717 if (!gGrid) {
718 Info("Connect", "Trying to connect to AliEn ...");
719 TGrid::Connect("alien://");
720 }
721 if (!gGrid || !gGrid->IsConnected()) {
722 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
723 return kFALSE;
724 }
725 fUser = gGrid->GetUser();
726 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
727 return kTRUE;
728}
729
730//______________________________________________________________________________
731void AliAnalysisAlien::CdWork()
732{
733// Check validity of alien workspace. Create directory if possible.
734 if (!Connect()) {
735 Error("CdWork", "Alien connection required");
736 return;
737 }
738 TString homedir = gGrid->GetHomeDirectory();
739 TString workdir = homedir + fGridWorkingDir;
923e2ca5 740 if (DirectoryExists(workdir)) {
741 gGrid->Cd(workdir);
742 return;
743 }
744 // Work directory not existing - create it
745 gGrid->Cd(homedir);
b93f8109 746 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 747 gGrid->Cd(fGridWorkingDir);
d3b18c4c 748 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 749 } else {
d3b18c4c 750 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 751 workdir.Data(), homedir.Data());
752 fGridWorkingDir = "";
753 }
c57f56b7 754}
755
348be253 756//______________________________________________________________________________
757Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
758{
759// Check if file copying is possible.
d3b18c4c 760 if (fProductionMode) return kTRUE;
348be253 761 if (!Connect()) {
762 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
763 return kFALSE;
764 }
d3339be3 765 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
766 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 767 // Check if alien_CLOSE_SE is defined
768 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
769 if (!closeSE.IsNull()) {
770 Info("CheckFileCopy", "Your current close storage is pointing to: \
771 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
772 } else {
773 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
774 }
775 // Check if grid directory exists.
776 if (!DirectoryExists(alienpath)) {
777 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
778 return kFALSE;
779 }
d3339be3 780 TFile f("plugin_test_copy", "RECREATE");
348be253 781 // User may not have write permissions to current directory
782 if (f.IsZombie()) {
783 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
784 gSystem->WorkingDirectory());
785 return kFALSE;
786 }
787 f.Close();
d3339be3 788 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
789 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
790 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 791 \n# 1. Make sure you have write permissions there. If this is the case: \
792 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
793 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
794 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
795 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
796 gSystem->Unlink(f.GetName());
797 return kFALSE;
798 }
799 gSystem->Unlink(f.GetName());
d3339be3 800 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
801 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 802 return kTRUE;
803}
804
c57f56b7 805//______________________________________________________________________________
806Bool_t AliAnalysisAlien::CheckInputData()
807{
808// Check validity of input data. If necessary, create xml files.
d3b18c4c 809 if (fProductionMode) return kTRUE;
a8739e8a 810 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 811 if (!fGridDataDir.Length()) {
812 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
813 return kFALSE;
814 }
9f020952 815 if (fMergeViaJDL) {
816 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
817 return kFALSE;
818 }
d2a409b2 819 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 820 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
821 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 822 return kTRUE;
c57f56b7 823 }
824 // Process declared files
a2f5fc01 825 Bool_t isCollection = kFALSE;
826 Bool_t isXml = kFALSE;
827 Bool_t useTags = kFALSE;
c57f56b7 828 Bool_t checked = kFALSE;
d3b18c4c 829 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 830 TString file;
831 TString workdir = gGrid->GetHomeDirectory();
832 workdir += fGridWorkingDir;
833 if (fInputFiles) {
834 TObjString *objstr;
835 TIter next(fInputFiles);
836 while ((objstr=(TObjString*)next())) {
837 file = workdir;
838 file += "/";
839 file += objstr->GetString();
840 // Store full lfn path
841 if (FileExists(file)) objstr->SetString(file);
842 else {
843 file = objstr->GetName();
844 if (!FileExists(objstr->GetName())) {
845 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
846 objstr->GetName(), workdir.Data());
847 return kFALSE;
848 }
849 }
850 Bool_t iscoll, isxml, usetags;
851 CheckDataType(file, iscoll, isxml, usetags);
852 if (!checked) {
853 checked = kTRUE;
a2f5fc01 854 isCollection = iscoll;
855 isXml = isxml;
856 useTags = usetags;
857 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 858 } else {
a2f5fc01 859 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 860 Error("CheckInputData", "Some conflict was found in the types of inputs");
861 return kFALSE;
862 }
863 }
864 }
865 }
866 // Process requested run numbers
a8739e8a 867 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 868 // Check validity of alien data directory
869 if (!fGridDataDir.Length()) {
870 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
871 return kFALSE;
872 }
923e2ca5 873 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 874 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
875 return kFALSE;
876 }
a2f5fc01 877 if (isCollection) {
c57f56b7 878 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
879 return kFALSE;
880 }
881
a2f5fc01 882 if (checked && !isXml) {
c57f56b7 883 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
884 return kFALSE;
885 }
886 // Check validity of run number(s)
887 TObjArray *arr;
888 TObjString *os;
74013535 889 TString format;
319593fb 890 Int_t nruns = 0;
904f9f5f 891 TString schunk, schunk2;
c57f56b7 892 TString path;
893 if (!checked) {
894 checked = kTRUE;
a2f5fc01 895 useTags = fDataPattern.Contains("tag");
896 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 897 }
a2f5fc01 898 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 899 Error("CheckInputData", "Cannot mix input files using/not using tags");
900 return kFALSE;
901 }
902 if (fRunNumbers.Length()) {
a8739e8a 903 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 904 arr = fRunNumbers.Tokenize(" ");
905 TIter next(arr);
906 while ((os=(TObjString*)next())) {
907 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 908 if (!DirectoryExists(path)) {
909 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 910 continue;
c57f56b7 911 }
912 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
913 TString msg = "\n##### file: ";
914 msg += path;
915 msg += " type: xml_collection;";
a2f5fc01 916 if (useTags) msg += " using_tags: Yes";
c57f56b7 917 else msg += " using_tags: No";
23329835 918 Info("CheckDataType", "%s", msg.Data());
319593fb 919 if (fNrunsPerMaster<2) {
d2a409b2 920 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 921 } else {
922 nruns++;
923 if (((nruns-1)%fNrunsPerMaster) == 0) {
924 schunk = os->GetString();
925 }
926 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
927 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 928 AddDataFile(schunk);
319593fb 929 }
c57f56b7 930 }
931 delete arr;
a8739e8a 932 } else {
933 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
934 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 935 format = Form("%%s/%s ", fRunPrefix.Data());
936 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 937 if (!DirectoryExists(path)) {
a8739e8a 938 continue;
939 }
74013535 940 format = Form("%%s/%s.xml", fRunPrefix.Data());
941 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 942 TString msg = "\n##### file: ";
943 msg += path;
944 msg += " type: xml_collection;";
a2f5fc01 945 if (useTags) msg += " using_tags: Yes";
a8739e8a 946 else msg += " using_tags: No";
23329835 947 Info("CheckDataType", "%s", msg.Data());
319593fb 948 if (fNrunsPerMaster<2) {
74013535 949 format = Form("%s.xml", fRunPrefix.Data());
950 AddDataFile(Form(format.Data(),irun));
319593fb 951 } else {
952 nruns++;
953 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 954 schunk = Form(fRunPrefix.Data(),irun);
319593fb 955 }
74013535 956 format = Form("_%s.xml", fRunPrefix.Data());
957 schunk2 = Form(format.Data(), irun);
319593fb 958 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 959 schunk += schunk2;
d2a409b2 960 AddDataFile(schunk);
319593fb 961 }
a8739e8a 962 }
904f9f5f 963 if (!fInputFiles) {
964 schunk += schunk2;
965 AddDataFile(schunk);
966 }
c57f56b7 967 }
968 return kTRUE;
969}
970
21ca8e59 971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *anchorfile)
973{
974// Copy data from the given grid directory according a pattern and make a local
975// dataset.
976 if (!Connect()) {
977 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
978 return kFALSE;
979 }
980 if (!DirectoryExists(griddir)) {
981 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
982 return kFALSE;
983 }
984 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
985 printf("Running command: %s\n", command.Data());
986 TGridResult *res = gGrid->Command(command);
987 Int_t nfound = res->GetEntries();
988 if (!nfound) {
989 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
990 return kFALSE;
991 }
992 printf("... found %d files. Copying locally ...\n", nfound);
993 // Copy files locally
994 ofstream out;
995 out.open(output, ios::out);
996 TMap *map;
997 TString turl, dirname, filename, temp;
998 TString cdir = gSystem->WorkingDirectory();
999 gSystem->MakeDirectory("data");
1000 gSystem->ChangeDirectory("data");
1001 for (Int_t i=0; i<nfound; i++) {
1002 map = (TMap*)res->At(i);
1003 turl = map->GetValue("turl")->GetName();
1004 filename = gSystem->BaseName(turl.Data());
1005 dirname = gSystem->DirName(turl.Data());
1006 dirname = gSystem->BaseName(dirname.Data());
1007 gSystem->MakeDirectory(dirname);
1008 if (TFile::Cp(turl, Form("file:./%s/%s", dirname.Data(), filename.Data()))) {
1009 if (strlen(anchorfile)) filename = Form("%s#%s", filename.Data(), anchorfile);
1010 out << cdir << "/data/" << Form("%s/%s", dirname.Data(), filename.Data()) << endl;
1011 }
1012 }
1013 gSystem->ChangeDirectory(cdir);
1014 delete res;
1015 return kTRUE;
1016}
1017
c57f56b7 1018//______________________________________________________________________________
1019Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1020{
1021// Create dataset for the grid data directory + run number.
3af861ff 1022 const Int_t gMaxEntries = 15000;
d3b18c4c 1023 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1024 if (!Connect()) {
1025 Error("CreateDataset", "Cannot create dataset with no grid connection");
1026 return kFALSE;
1027 }
1028
1029 // Cd workspace
d3b18c4c 1030 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1031 TString workdir = gGrid->GetHomeDirectory();
1032 workdir += fGridWorkingDir;
1033
1034 // Compose the 'find' command arguments
74013535 1035 TString format;
c57f56b7 1036 TString command;
1037 TString options = "-x collection ";
bb885a9e 1038 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1039 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1040 TString conditions = "";
3af861ff 1041 Int_t nstart = 0;
1042 Int_t ncount = 0;
1043 Int_t stage = 0;
c57f56b7 1044 TString file;
1045 TString path;
319593fb 1046 Int_t nruns = 0;
904f9f5f 1047 TString schunk, schunk2;
ab254fd1 1048 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1049 if (!fRunNumbers.Length() && !fRunRange[0]) {
1050 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1051 // Make a single data collection from data directory.
1052 path = fGridDataDir;
923e2ca5 1053 if (!DirectoryExists(path)) {
d2a409b2 1054 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1055 return kFALSE;
1056 }
923e2ca5 1057// CdWork();
d2a409b2 1058 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1059 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1060 while (1) {
1061 ncount = 0;
1062 stage++;
1063 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1064 command = "find ";
1065 command += Form("%s -o %d ",options.Data(), nstart);
1066 command += path;
1067 command += " ";
1068 command += pattern;
1069 command += conditions;
1070 printf("command: %s\n", command.Data());
1071 TGridResult *res = gGrid->Command(command);
1072 if (res) delete res;
1073 // Write standard output to file
1074 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1075 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1076 Bool_t nullFile = kFALSE;
1077 if (!hasGrep) {
1078 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1079 } else {
1080 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1081 if (nullFile) {
1082 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1083 gSystem->Exec("rm -f __tmp*");
1084 return kFALSE;
1085 }
1086 TString line;
1087 ifstream in;
1088 in.open("__tmp__");
1089 in >> line;
1090 in.close();
1091 gSystem->Exec("rm -f __tmp__");
1092 ncount = line.Atoi();
1093 }
1094 }
1095 if (ncount == gMaxEntries) {
1096 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1097 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1098 if (!cbase) cbase = cadd;
1099 else {
1100 cbase->Add(cadd);
1101 delete cadd;
defd7a3a 1102 }
3af861ff 1103 nstart += ncount;
1104 } else {
1105 if (cbase) {
1106 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1107 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1108 cbase->Add(cadd);
1109 delete cadd;
1110 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1111 delete cbase; cbase = 0;
1112 } else {
1113 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1114 }
1115 gSystem->Exec("rm -f __tmp*");
1116 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1117 break;
1118 }
0f389141 1119 }
1120 Bool_t fileExists = FileExists(file);
1121 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1122 // Copy xml file to alien space
0f389141 1123 if (fileExists) gGrid->Rm(file);
d2a409b2 1124 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1125 if (!FileExists(file)) {
1126 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1127 return kFALSE;
1128 }
1129 // Update list of files to be processed.
1130 }
1131 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1132 return kTRUE;
1133 }
c57f56b7 1134 // Several runs
a2f5fc01 1135 Bool_t nullResult = kTRUE;
a8739e8a 1136 if (fRunNumbers.Length()) {
1137 TObjArray *arr = fRunNumbers.Tokenize(" ");
1138 TObjString *os;
1139 TIter next(arr);
1140 while ((os=(TObjString*)next())) {
3af861ff 1141 nstart = 0;
1142 stage = 0;
dc030a24 1143 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1144 if (!DirectoryExists(path)) continue;
1145// CdWork();
a8739e8a 1146 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1147 else file = Form("%s.xml", os->GetString().Data());
319593fb 1148 // If local collection file does not exist, create it via 'find' command.
3af861ff 1149 while (1) {
1150 ncount = 0;
1151 stage++;
1152 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1153 command = "find ";
1154 command += Form("%s -o %d ",options.Data(), nstart);
1155 command += path;
1156 command += pattern;
1157 command += conditions;
1158 TGridResult *res = gGrid->Command(command);
1159 if (res) delete res;
1160 // Write standard output to file
1161 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1162 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1163 Bool_t nullFile = kFALSE;
1164 if (!hasGrep) {
1165 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1166 } else {
1167 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1168 if (nullFile) {
1169 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1170 gSystem->Exec("rm -f __tmp*");
1171 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1172 break;
3af861ff 1173 }
1174 TString line;
1175 ifstream in;
1176 in.open("__tmp__");
1177 in >> line;
1178 in.close();
1179 gSystem->Exec("rm -f __tmp__");
1180 ncount = line.Atoi();
1181 }
1182 nullResult = kFALSE;
1183 }
1184 if (ncount == gMaxEntries) {
1185 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1186 if (fNrunsPerMaster > 1) {
1187 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1188 file.Data(),gMaxEntries);
1189 return kFALSE;
1190 }
1191 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1192 if (!cbase) cbase = cadd;
1193 else {
1194 cbase->Add(cadd);
1195 delete cadd;
defd7a3a 1196 }
3af861ff 1197 nstart += ncount;
1198 } else {
c89a84bc 1199 if (cbase && fNrunsPerMaster<2) {
3af861ff 1200 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1201 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1202 cbase->Add(cadd);
1203 delete cadd;
1204 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1205 delete cbase; cbase = 0;
1206 } else {
1207 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1208 }
1209 gSystem->Exec("rm -f __tmp*");
1210 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1211 break;
d3339be3 1212 }
3af861ff 1213 }
a8739e8a 1214 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1215 // Check if there is one run per master job.
1216 if (fNrunsPerMaster<2) {
1217 if (FileExists(file)) {
0f389141 1218 if (fOverwriteMode) gGrid->Rm(file);
1219 else {
1220 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1221 continue;
1222 }
319593fb 1223 }
1224 // Copy xml file to alien space
1225 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1226 if (!FileExists(file)) {
1227 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1228 delete arr;
1229 return kFALSE;
1230 }
1231 } else {
1232 nruns++;
1233 if (((nruns-1)%fNrunsPerMaster) == 0) {
1234 schunk = os->GetString();
1235 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1236 } else {
1237 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1238 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1239 cbase->Add(cadd);
1240 delete cadd;
1241 }
1242 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1243 continue;
1244 }
1245 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1246 if (FileExists(schunk)) {
1247 if (fOverwriteMode) gGrid->Rm(file);
1248 else {
1249 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1250 continue;
1251 }
319593fb 1252 }
84fcd93f 1253 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1254 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1255 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1256 if (!FileExists(schunk)) {
1257 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1258 delete arr;
1259 return kFALSE;
1260 }
d3339be3 1261 }
a8739e8a 1262 }
1263 delete arr;
a2f5fc01 1264 if (nullResult) {
d3339be3 1265 Error("CreateDataset", "No valid dataset corresponding to the query!");
1266 return kFALSE;
1267 }
a8739e8a 1268 } else {
1269 // Process a full run range.
1270 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1271 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1272 nstart = 0;
1273 stage = 0;
74013535 1274 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1275 if (!DirectoryExists(path)) continue;
1276// CdWork();
74013535 1277 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1278 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1279 else file = Form(format.Data(), irun);
0f389141 1280 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1281 if (fOverwriteMode) gGrid->Rm(file);
1282 else {
1283 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1284 continue;
1285 }
a8739e8a 1286 }
319593fb 1287 // If local collection file does not exist, create it via 'find' command.
3af861ff 1288 while (1) {
1289 ncount = 0;
1290 stage++;
1291 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1292 command = "find ";
1293 command += Form("%s -o %d ",options.Data(), nstart);
1294 command += path;
1295 command += pattern;
1296 command += conditions;
1297 TGridResult *res = gGrid->Command(command);
1298 if (res) delete res;
1299 // Write standard output to file
1300 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1301 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1302 Bool_t nullFile = kFALSE;
1303 if (!hasGrep) {
1304 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1305 } else {
1306 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1307 if (nullFile) {
1308 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1309 gSystem->Exec("rm -f __tmp*");
1604c9ed 1310 break;
3af861ff 1311 }
1312 TString line;
1313 ifstream in;
1314 in.open("__tmp__");
1315 in >> line;
1316 in.close();
1317 gSystem->Exec("rm -f __tmp__");
1318 ncount = line.Atoi();
1319 }
1320 nullResult = kFALSE;
1321 }
1322 if (ncount == gMaxEntries) {
1323 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1324 if (fNrunsPerMaster > 1) {
1325 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1326 file.Data(),gMaxEntries);
1327 return kFALSE;
1328 }
1329 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1330 if (!cbase) cbase = cadd;
1331 else {
1332 cbase->Add(cadd);
1333 delete cadd;
defd7a3a 1334 }
3af861ff 1335 nstart += ncount;
1336 } else {
c89a84bc 1337 if (cbase && fNrunsPerMaster<2) {
3af861ff 1338 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1339 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1340 cbase->Add(cadd);
1341 delete cadd;
1342 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1343 delete cbase; cbase = 0;
1344 } else {
1345 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1346 }
1347 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1348 break;
d3339be3 1349 }
319593fb 1350 }
a8739e8a 1351 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1352 // Check if there is one run per master job.
1353 if (fNrunsPerMaster<2) {
1354 if (FileExists(file)) {
0f389141 1355 if (fOverwriteMode) gGrid->Rm(file);
1356 else {
1357 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1358 continue;
1359 }
319593fb 1360 }
1361 // Copy xml file to alien space
1362 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1363 if (!FileExists(file)) {
1364 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1365 return kFALSE;
1366 }
1367 } else {
1368 nruns++;
95e5b448 1369 // Check if the collection for the chunk exist locally.
1370 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1371 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1372 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1373 else continue;
1374 }
84fcd93f 1375 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1376 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1377 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1378 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1379 } else {
1380 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1381 cbase->Add(cadd);
1382 delete cadd;
1383 }
74013535 1384 format = Form("%%s_%s.xml", fRunPrefix.Data());
1385 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1386 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1387 continue;
1388 }
904f9f5f 1389 schunk = schunk2;
319593fb 1390 if (FileExists(schunk)) {
0f389141 1391 if (fOverwriteMode) gGrid->Rm(schunk);
1392 else {
1393 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1394 continue;
1395 }
319593fb 1396 }
84fcd93f 1397 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1398 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1399 if (FileExists(schunk)) {
0f389141 1400 if (fOverwriteMode) gGrid->Rm(schunk);
1401 else {
1402 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1403 continue;
1404 }
95e5b448 1405 }
319593fb 1406 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1407 if (!FileExists(schunk)) {
1408 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1409 return kFALSE;
1410 }
1411 }
c57f56b7 1412 }
a2f5fc01 1413 if (nullResult) {
d3339be3 1414 Error("CreateDataset", "No valid dataset corresponding to the query!");
1415 return kFALSE;
1416 }
a8739e8a 1417 }
c57f56b7 1418 return kTRUE;
1419}
1420
1421//______________________________________________________________________________
1422Bool_t AliAnalysisAlien::CreateJDL()
1423{
1424// Generate a JDL file according to current settings. The name of the file is
1425// specified by fJDLName.
1426 Bool_t error = kFALSE;
1427 TObjArray *arr = 0;
1428 Bool_t copy = kTRUE;
d3b18c4c 1429 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1430 Bool_t generate = kTRUE;
1431 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1432 if (!Connect()) {
1433 Error("CreateJDL", "Alien connection required");
1434 return kFALSE;
1435 }
1436 // Check validity of alien workspace
d3b18c4c 1437 TString workdir;
1438 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1439 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1440 workdir += fGridWorkingDir;
1441 if (generate) {
1442 TObjString *os;
1443 if (!fInputFiles) {
1444 Error("CreateJDL()", "Define some input files for your analysis.");
1445 error = kTRUE;
1446 }
1447 // Compose list of input files
1448 // Check if output files were defined
1449 if (!fOutputFiles.Length()) {
1450 Error("CreateJDL", "You must define at least one output file");
1451 error = kTRUE;
1452 }
1453 // Check if an output directory was defined and valid
1454 if (!fGridOutputDir.Length()) {
1455 Error("CreateJDL", "You must define AliEn output directory");
1456 error = kTRUE;
1457 } else {
d3b18c4c 1458 if (!fProductionMode) {
1459 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1460 if (!DirectoryExists(fGridOutputDir)) {
1461 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1462 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1463 } else {
1464 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1465 // error = kTRUE;
1466 }
67a3b254 1467 } else {
1468 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1469 }
1470 gGrid->Cd(workdir);
1471 }
c57f56b7 1472 }
1473 // Exit if any error up to now
1474 if (error) return kFALSE;
1475 // Set JDL fields
0f389141 1476 if (!fUser.IsNull()) {
1477 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1478 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1479 }
1480 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1481 TString mergeExec = fExecutable;
1482 mergeExec.ReplaceAll(".sh", "_merge.sh");
1483 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1484 mergeExec.ReplaceAll(".sh", ".C");
1485 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1486 if (!fArguments.IsNull())
1487 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1488 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1489 else {
1490 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1491 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1492 }
b3e07543 1493
1f0d1ca2 1494 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1495 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1496 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1497 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1498
0f389141 1499 if (fMaxInitFailed > 0) {
c57f56b7 1500 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1501 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1502 }
1503 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1504 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1505 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1506 }
1507 if (!IsOneStageMerging()) {
f47d5cb4 1508 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1509 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1510 }
1511 if (fSplitMode.Length()) {
c57f56b7 1512 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1513 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1514 }
1515 fMergingJDL->SetValue("Split", "\"se\"");
1516 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1517 if (!fAliROOTVersion.IsNull()) {
1518 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1519 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1520 }
1521 if (!fROOTVersion.IsNull()) {
c57f56b7 1522 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1523 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1524 }
1525 if (!fAPIVersion.IsNull()) {
c57f56b7 1526 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1527 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1528 }
648174cf 1529 if (!fExternalPackages.IsNull()) {
1530 arr = fExternalPackages.Tokenize(" ");
1531 TIter next(arr);
1532 while ((os=(TObjString*)next())) {
1533 TString pkgname = os->GetString();
1534 Int_t index = pkgname.Index("::");
1535 TString pkgversion = pkgname(index+2, pkgname.Length());
1536 pkgname.Remove(index);
1537 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1538 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1539 }
1540 delete arr;
1541 }
0f389141 1542 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1543 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1544 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1545 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1546 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1547 TString analysisFile = fExecutable;
1548 analysisFile.ReplaceAll(".sh", ".root");
1549 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1550 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1551 if (fAdditionalLibs.Length()) {
1552 arr = fAdditionalLibs.Tokenize(" ");
1553 TIter next(arr);
1554 while ((os=(TObjString*)next())) {
1555 if (os->GetString().Contains(".so")) continue;
1556 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1557 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1558 }
1559 delete arr;
1560 }
4e5c5506 1561 if (fPackages) {
1562 TIter next(fPackages);
1563 TObject *obj;
0f389141 1564 while ((obj=next())) {
4e5c5506 1565 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1566 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1567 }
4e5c5506 1568 }
c57f56b7 1569 if (fOutputArchive.Length()) {
1570 arr = fOutputArchive.Tokenize(" ");
1571 TIter next(arr);
0f389141 1572 Bool_t first = kTRUE;
1573 const char *comment = "Files to be archived";
1574 const char *comment1 = comment;
1575 while ((os=(TObjString*)next())) {
1576 if (!first) comment = NULL;
1577 if (!os->GetString().Contains("@") && fCloseSE.Length())
1578 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1579 else
1580 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1581 first = kFALSE;
1582 }
c57f56b7 1583 delete arr;
f790bc1b 1584 // Output archive for the merging jdl
1585 TString outputArchive;
1586 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1587 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1588 // Add normal output files, extra files + terminate files
1589 TString files = GetListOfFiles("outextter");
1590 // Do not register merge excludes
1591 if (!fMergeExcludes.IsNull()) {
1592 arr = fMergeExcludes.Tokenize(" ");
1593 TIter next1(arr);
1594 while ((os=(TObjString*)next1())) {
1595 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1596 files.ReplaceAll(os->GetString(),"");
1597 }
66d71516 1598 delete arr;
0f389141 1599 }
f790bc1b 1600 files.ReplaceAll(".root", "*.root");
b83f84f0 1601 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1602 } else {
95f84779 1603 TString files = fOutputArchive;
1604 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1605 outputArchive = files;
f790bc1b 1606 }
0f389141 1607 arr = outputArchive.Tokenize(" ");
705adb3e 1608 TIter next2(arr);
0f389141 1609 comment = comment1;
1610 first = kTRUE;
705adb3e 1611 while ((os=(TObjString*)next2())) {
0f389141 1612 if (!first) comment = NULL;
0cdf65a8 1613 TString currentfile = os->GetString();
0cdf65a8 1614 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1615 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1616 else
0cdf65a8 1617 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1618 first = kFALSE;
1619 }
1620 delete arr;
c57f56b7 1621 }
149d288c 1622 arr = fOutputFiles.Tokenize(",");
c57f56b7 1623 TIter next(arr);
0f389141 1624 Bool_t first = kTRUE;
f790bc1b 1625 const char *comment = "Files to be saved";
43da816a 1626 while ((os=(TObjString*)next())) {
e1eaf596 1627 // Ignore ouputs in jdl that are also in outputarchive
1628 TString sout = os->GetString();
f790bc1b 1629 sout.ReplaceAll("*", "");
1630 sout.ReplaceAll(".root", "");
e1eaf596 1631 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1632 if (fOutputArchive.Contains(sout)) continue;
0f389141 1633 if (!first) comment = NULL;
43da816a 1634 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1635 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1636 else
0f389141 1637 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1638 first = kFALSE;
f790bc1b 1639 if (fMergeExcludes.Contains(sout)) continue;
1640 if (!os->GetString().Contains("@") && fCloseSE.Length())
1641 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1642 else
1643 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1644 }
c57f56b7 1645 delete arr;
0f389141 1646 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1647 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1648 TString validationScript = fValidationScript;
0f389141 1649 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1650 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1651 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1652 if (fMasterResubmitThreshold) {
1653 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1654 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1655 }
d2a409b2 1656 // Write a jdl with 2 input parameters: collection name and output dir name.
1657 WriteJDL(copy);
c57f56b7 1658 }
1659 // Copy jdl to grid workspace
a8739e8a 1660 if (copy) {
b5e4aaa7 1661 // Check if an output directory was defined and valid
1662 if (!fGridOutputDir.Length()) {
1663 Error("CreateJDL", "You must define AliEn output directory");
1664 return kFALSE;
1665 } else {
1666 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1667 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1668 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1669 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1670 } else {
1671 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1672 return kFALSE;
1673 }
1674 }
1675 gGrid->Cd(workdir);
1676 }
648174cf 1677 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1678 TString mergeJDLName = fExecutable;
1679 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1680 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1681 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1682 if (fProductionMode) {
648174cf 1683 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1684 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1685 }
648174cf 1686 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1687 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1688 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1689 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1690 Fatal("","Terminating");
1691// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1692 if (fMergeViaJDL) {
1693 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1694// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1695 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1696 Fatal("","Terminating");
0f389141 1697 }
648174cf 1698 }
c57f56b7 1699 if (fAdditionalLibs.Length()) {
e7c71df0 1700 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1701 TObjString *os;
1702 TIter next(arr);
1703 while ((os=(TObjString*)next())) {
c57f56b7 1704 if (os->GetString().Contains(".so")) continue;
4e5c5506 1705 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1706 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1707// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1708 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1709 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1710 Fatal("","Terminating");
c57f56b7 1711 }
1712 delete arr;
1713 }
4e5c5506 1714 if (fPackages) {
1715 TIter next(fPackages);
1716 TObject *obj;
1717 while ((obj=next())) {
fdbbc7be 1718 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1719 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1720// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1721 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1722 Form("%s/%s", workdir.Data(), obj->GetName())))
1723 Fatal("","Terminating");
4e5c5506 1724 }
1725 }
c57f56b7 1726 }
1727 return kTRUE;
1728}
1729
a8739e8a 1730//______________________________________________________________________________
d2a409b2 1731Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1732{
1733// Writes one or more JDL's corresponding to findex. If findex is negative,
1734// all run numbers are considered in one go (jdl). For non-negative indices
1735// they correspond to the indices in the array fInputFiles.
1736 if (!fInputFiles) return kFALSE;
d3b18c4c 1737 TObject *os;
1738 TString workdir;
1739 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1740 workdir += fGridWorkingDir;
f47d5cb4 1741 TString stageName = "$2";
1742 if (fProductionMode) stageName = "$4";
1743 if (!fMergeDirName.IsNull()) {
1744 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1745 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1746 } else {
1747 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1748 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1749 }
d3b18c4c 1750 if (fProductionMode) {
a8739e8a 1751 TIter next(fInputFiles);
b3e07543 1752 while ((os=next())) {
d3b18c4c 1753 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1754 }
56754bf6 1755 if (!fOutputToRunNo)
1756 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1757 else
1758 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1759 } else {
1760 if (!fRunNumbers.Length() && !fRunRange[0]) {
1761 // One jdl with no parameters in case input data is specified by name.
1762 TIter next(fInputFiles);
1763 while ((os=next()))
1764 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1765 if (!fOutputSingle.IsNull())
1766 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1767 else {
1768 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1769 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1770 }
1771 } else {
1772 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1773 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1774 if (!fOutputSingle.IsNull()) {
1775 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1776 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1777 } else {
1778 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1779 }
1780 }
a8739e8a 1781 }
1782
a8739e8a 1783 // Generate the JDL as a string
1784 TString sjdl = fGridJDL->Generate();
0f389141 1785 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1786 // Final merge jdl
1787 if (!fMergeDirName.IsNull()) {
1788 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1789 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1790 } else {
1791 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1792 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1793 }
b3e07543 1794 TString sjdl2 = fMergingJDL->Generate();
1795 Int_t index, index1;
a8739e8a 1796 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1797 sjdl.ReplaceAll("(member", "\n (member");
1798 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1799 sjdl.ReplaceAll("{", "{\n ");
1800 sjdl.ReplaceAll("};", "\n};");
1801 sjdl.ReplaceAll("{\n \n", "{\n");
1802 sjdl.ReplaceAll("\n\n", "\n");
1803 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1804 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1805 sjdl1.ReplaceAll("(member", "\n (member");
1806 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1807 sjdl1.ReplaceAll("{", "{\n ");
1808 sjdl1.ReplaceAll("};", "\n};");
1809 sjdl1.ReplaceAll("{\n \n", "{\n");
1810 sjdl1.ReplaceAll("\n\n", "\n");
1811 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1812 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1813 sjdl2.ReplaceAll("(member", "\n (member");
1814 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1815 sjdl2.ReplaceAll("{", "{\n ");
1816 sjdl2.ReplaceAll("};", "\n};");
1817 sjdl2.ReplaceAll("{\n \n", "{\n");
1818 sjdl2.ReplaceAll("\n\n", "\n");
1819 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1820 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1821 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1822 index = sjdl.Index("JDLVariables");
1823 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1824 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1825 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1826 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1827 index = fJobTag.Index(":");
1828 if (index < 0) index = fJobTag.Length();
1829 TString jobTag = fJobTag;
f47d5cb4 1830 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1831 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1832 if (fProductionMode) {
1833 sjdl1.Prepend("# Generated merging jdl (production mode) \
1834 \n# $1 = full alien path to output directory to be merged \
1835 \n# $2 = train number \
1836 \n# $3 = production (like LHC10b) \
1837 \n# $4 = merging stage \
1838 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1839 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1840 sjdl2.Prepend("# Generated merging jdl \
1841 \n# $1 = full alien path to output directory to be merged \
1842 \n# $2 = train number \
1843 \n# $3 = production (like LHC10b) \
1844 \n# $4 = merging stage \
1845 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1846 } else {
1847 sjdl1.Prepend("# Generated merging jdl \
1848 \n# $1 = full alien path to output directory to be merged \
1849 \n# $2 = merging stage \
03335324 1850 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1851 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1852 sjdl2.Prepend("# Generated merging jdl \
1853 \n# $1 = full alien path to output directory to be merged \
1854 \n# $2 = merging stage \
03335324 1855 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1856 }
0f389141 1857 index = sjdl1.Index("JDLVariables");
1858 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1859 index = sjdl2.Index("JDLVariables");
1860 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1861 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1862 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1863 index = sjdl2.Index("Split =");
1864 if (index>=0) {
1865 index1 = sjdl2.Index("\n", index);
1866 sjdl2.Remove(index, index1-index+1);
1867 }
1868 index = sjdl2.Index("SplitMaxInputFileNumber");
1869 if (index>=0) {
1870 index1 = sjdl2.Index("\n", index);
1871 sjdl2.Remove(index, index1-index+1);
1872 }
1873 index = sjdl2.Index("InputDataCollection");
1874 if (index>=0) {
1875 index1 = sjdl2.Index(";", index);
1876 sjdl2.Remove(index, index1-index+1);
1877 }
1878 index = sjdl2.Index("InputDataListFormat");
1879 if (index>=0) {
1880 index1 = sjdl2.Index("\n", index);
1881 sjdl2.Remove(index, index1-index+1);
1882 }
1883 index = sjdl2.Index("InputDataList");
1884 if (index>=0) {
1885 index1 = sjdl2.Index("\n", index);
1886 sjdl2.Remove(index, index1-index+1);
1887 }
f47d5cb4 1888 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1889 // Write jdl to file
a8739e8a 1890 ofstream out;
d2a409b2 1891 out.open(fJDLName.Data(), ios::out);
a8739e8a 1892 if (out.bad()) {
d3b18c4c 1893 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1894 return kFALSE;
1895 }
1896 out << sjdl << endl;
b3e07543 1897 out.close();
0f389141 1898 TString mergeJDLName = fExecutable;
1899 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1900 if (fMergeViaJDL) {
1901 ofstream out1;
1902 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1903 if (out1.bad()) {
d3b18c4c 1904 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1905 return kFALSE;
1906 }
1907 out1 << sjdl1 << endl;
b3e07543 1908 out1.close();
1909 ofstream out2;
1910 TString finalJDL = mergeJDLName;
1911 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1912 out2.open(finalJDL.Data(), ios::out);
1913 if (out2.bad()) {
1914 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1915 return kFALSE;
1916 }
1917 out2 << sjdl2 << endl;
1918 out2.close();
0f389141 1919 }
a8739e8a 1920
1921 // Copy jdl to grid workspace
1922 if (!copy) {
d3b18c4c 1923 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1924 } else {
d2a409b2 1925 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1926 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1927 TString finalJDL = mergeJDLName;
1928 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1929 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1930 if (fProductionMode) {
b5fe9cba 1931 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1932 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1933 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1934 }
d2a409b2 1935 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1936 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1937 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1938 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1939// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1940 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
1941 Fatal("","Terminating");
0f389141 1942 if (fMergeViaJDL) {
b3e07543 1943 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1944// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1945// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
1946 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
1947 Fatal("","Terminating");
1948 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
1949 Fatal("","Terminating");
0f389141 1950 }
a8739e8a 1951 }
1952 return kTRUE;
1953}
1954
c57f56b7 1955//______________________________________________________________________________
5513444a 1956Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1957{
1958// Returns true if file exists.
5513444a 1959 if (!gGrid) return kFALSE;
2ea55496 1960 TString slfn = lfn;
1961 slfn.ReplaceAll("alien://","");
1962 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1963 if (!res) return kFALSE;
1964 TMap *map = dynamic_cast<TMap*>(res->At(0));
1965 if (!map) {
1966 delete res;
1967 return kFALSE;
1968 }
1969 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1970 if (!objs || !objs->GetString().Length()) {
1971 delete res;
1972 return kFALSE;
1973 }
1974 delete res;
1975 return kTRUE;
1976}
1977
923e2ca5 1978//______________________________________________________________________________
1979Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1980{
1981// Returns true if directory exists. Can be also a path.
1982 if (!gGrid) return kFALSE;
1983 // Check if dirname is a path
1984 TString dirstripped = dirname;
1985 dirstripped = dirstripped.Strip();
1986 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1987 TString dir = gSystem->BaseName(dirstripped);
1988 dir += "/";
1989 TString path = gSystem->DirName(dirstripped);
1990 TGridResult *res = gGrid->Ls(path, "-F");
1991 if (!res) return kFALSE;
1992 TIter next(res);
1993 TMap *map;
1994 TObject *obj;
1995 while ((map=dynamic_cast<TMap*>(next()))) {
1996 obj = map->GetValue("name");
1997 if (!obj) break;
1998 if (dir == obj->GetName()) {
1999 delete res;
2000 return kTRUE;
2001 }
2002 }
2003 delete res;
2004 return kFALSE;
2005}
2006
c57f56b7 2007//______________________________________________________________________________
a2f5fc01 2008void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2009{
2010// Check input data type.
a2f5fc01 2011 isCollection = kFALSE;
2012 isXml = kFALSE;
2013 useTags = kFALSE;
c57f56b7 2014 if (!gGrid) {
2015 Error("CheckDataType", "No connection to grid");
2016 return;
2017 }
a2f5fc01 2018 isCollection = IsCollection(lfn);
c57f56b7 2019 TString msg = "\n##### file: ";
2020 msg += lfn;
a2f5fc01 2021 if (isCollection) {
c57f56b7 2022 msg += " type: raw_collection;";
2023 // special treatment for collections
a2f5fc01 2024 isXml = kFALSE;
c57f56b7 2025 // check for tag files in the collection
2026 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2027 if (!res) {
2028 msg += " using_tags: No (unknown)";
23329835 2029 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2030 return;
2031 }
2032 const char* typeStr = res->GetKey(0, "origLFN");
2033 if (!typeStr || !strlen(typeStr)) {
2034 msg += " using_tags: No (unknown)";
23329835 2035 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2036 return;
2037 }
2038 TString file = typeStr;
a2f5fc01 2039 useTags = file.Contains(".tag");
2040 if (useTags) msg += " using_tags: Yes";
c57f56b7 2041 else msg += " using_tags: No";
23329835 2042 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2043 return;
2044 }
2045 TString slfn(lfn);
2046 slfn.ToLower();
a2f5fc01 2047 isXml = slfn.Contains(".xml");
2048 if (isXml) {
c57f56b7 2049 // Open xml collection and check if there are tag files inside
2050 msg += " type: xml_collection;";
2051 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2052 if (!coll) {
2053 msg += " using_tags: No (unknown)";
23329835 2054 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2055 return;
2056 }
2057 TMap *map = coll->Next();
2058 if (!map) {
2059 msg += " using_tags: No (unknown)";
23329835 2060 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2061 return;
2062 }
2063 map = (TMap*)map->GetValue("");
2064 TString file;
2065 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2066 useTags = file.Contains(".tag");
c57f56b7 2067 delete coll;
a2f5fc01 2068 if (useTags) msg += " using_tags: Yes";
c57f56b7 2069 else msg += " using_tags: No";
23329835 2070 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2071 return;
2072 }
a2f5fc01 2073 useTags = slfn.Contains(".tag");
c57f56b7 2074 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2075 else msg += " type: unknown file;";
a2f5fc01 2076 if (useTags) msg += " using_tags: Yes";
c57f56b7 2077 else msg += " using_tags: No";
23329835 2078 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2079}
2080
4e5c5506 2081//______________________________________________________________________________
2082void AliAnalysisAlien::EnablePackage(const char *package)
2083{
2084// Enables a par file supposed to exist in the current directory.
2085 TString pkg(package);
2086 pkg.ReplaceAll(".par", "");
2087 pkg += ".par";
2088 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2089 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2090 return;
2091 }
2092 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2093 Info("EnablePackage", "AliEn plugin will use .par packages");
2094 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2095 if (!fPackages) {
2096 fPackages = new TObjArray();
2097 fPackages->SetOwner();
2098 }
2099 fPackages->Add(new TObjString(pkg));
2100}
2101
3bdcb562 2102//______________________________________________________________________________
2103TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2104{
2105// Make a tree from files having the location specified in fFileForTestMode.
2106// Inspired from JF's CreateESDChain.
2107 if (fFileForTestMode.IsNull()) {
2108 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2109 return NULL;
2110 }
2111 if (gSystem->AccessPathName(fFileForTestMode)) {
2112 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2113 return NULL;
2114 }
2115 // Open the file
2116 ifstream in;
2117 in.open(fFileForTestMode);
2118 Int_t count = 0;
2119 // Read the input list of files and add them to the chain
2120 TString line;
2121 TChain *chain = new TChain(treeName);
2122 while (in.good())
2123 {
2124 in >> line;
2125 if (line.IsNull()) continue;
2126 if (count++ == fNtestFiles) break;
2127 TString esdFile(line);
2128 TFile *file = TFile::Open(esdFile);
2129 if (file) {
2130 if (!file->IsZombie()) chain->Add(esdFile);
2131 file->Close();
2132 } else {
2133 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2134 }
2135 }
2136 in.close();
2137 if (!chain->GetListOfFiles()->GetEntries()) {
2138 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2139 delete chain;
2140 return NULL;
2141 }
2142// chain->ls();
2143 return chain;
2144}
2145
d2a409b2 2146//______________________________________________________________________________
2147const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2148{
2149// Get job status for all jobs with jobid>jobidstart.
2150 static char mstatus[20];
2151 mstatus[0] = '\0';
2152 nrunning = 0;
2153 nwaiting = 0;
2154 nerror = 0;
2155 ndone = 0;
2156 TGridJobStatusList *list = gGrid->Ps("");
2157 if (!list) return mstatus;
2158 Int_t nentries = list->GetSize();
2159 TGridJobStatus *status;
2160 Int_t pid;
2161 for (Int_t ijob=0; ijob<nentries; ijob++) {
2162 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2163 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2164 if (pid<jobidstart) continue;
2165 if (pid == lastid) {
8ddad121 2166 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2167 }
2168 switch (status->GetStatus()) {
2169 case TGridJobStatus::kWAITING:
2170 nwaiting++; break;
2171 case TGridJobStatus::kRUNNING:
2172 nrunning++; break;
2173 case TGridJobStatus::kABORTED:
2174 case TGridJobStatus::kFAIL:
2175 case TGridJobStatus::kUNKNOWN:
2176 nerror++; break;
2177 case TGridJobStatus::kDONE:
2178 ndone++;
2179 }
2180 }
2181 list->Delete();
2182 delete list;
2183 return mstatus;
2184}
2185
c57f56b7 2186//______________________________________________________________________________
2187Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2188{
2189// Returns true if file is a collection. Functionality duplicated from
2190// TAlien::Type() because we don't want to directly depend on TAlien.
2191 if (!gGrid) {
2192 Error("IsCollection", "No connection to grid");
2193 return kFALSE;
2194 }
2195 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2196 if (!res) return kFALSE;
2197 const char* typeStr = res->GetKey(0, "type");
2198 if (!typeStr || !strlen(typeStr)) return kFALSE;
2199 if (!strcmp(typeStr, "collection")) return kTRUE;
2200 delete res;
2201 return kFALSE;
2202}
2203
fe2d7fc2 2204//______________________________________________________________________________
2205Bool_t AliAnalysisAlien::IsSingleOutput() const
2206{
2207// Check if single-ouput option is on.
2208 return (!fOutputSingle.IsNull());
2209}
2210
16a4353c 2211//______________________________________________________________________________
2212void AliAnalysisAlien::Print(Option_t *) const
2213{
2214// Print current plugin settings.
84fcd93f 2215 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2216 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2217 if (mgr && mgr->IsProofMode()) {
2218 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2219 if (TestBit(AliAnalysisGrid::kTest))
2220 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2221 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2222 if (!fProofDataSet.IsNull())
2223 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2224 if (fProofReset==1)
2225 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2226 if (fProofReset>1)
2227 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2228 if (!fRootVersionForProof.IsNull())
2229 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
2230 else
2231 printf("= ROOT version requested________________________ default\n");
2232 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2233 if (!fAliRootMode.IsNull())
2234 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2235 if (fNproofWorkers)
2236 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2237 if (fNproofWorkersPerSlave)
2238 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2239 if (TestSpecialBit(kClearPackages))
2240 printf("= ClearPackages requested...\n");
2241 if (fIncludePath.Data())
2242 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2243 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2244 if (fPackages && fPackages->GetEntries()) {
2245 TIter next(fPackages);
2246 TObject *obj;
2247 TString list;
2248 while ((obj=next())) list += obj->GetName();
2249 printf("= Par files to be used: ________________________ %s\n", list.Data());
2250 }
2251 if (TestSpecialBit(kProofConnectGrid))
2252 printf("= Requested PROOF connection to grid\n");
2253 return;
2254 }
e1c22e21 2255 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2256 if (fOverwriteMode) {
2257 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2258 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2259 }
348be253 2260 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2261 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2262 printf("= Production mode:______________________________ %d\n", fProductionMode);
2263 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2264 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2265 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2266 if (fUser.Length())
84fcd93f 2267 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2268 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2269 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2270 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2271 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2272 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2273 if (fRunNumbers.Length())
84fcd93f 2274 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2275 if (fRunRange[0])
74013535 2276 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2277 if (!fRunRange[0] && !fRunNumbers.Length()) {
2278 TIter next(fInputFiles);
2279 TObject *obj;
2280 TString list;
2281 while ((obj=next())) list += obj->GetName();
84fcd93f 2282 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2283 }
2284 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2285 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2286 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2287 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2288 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 2289 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2290 printf("=====================================================================\n");
2291 printf("= Job price: ___________________________________ %d\n", fPrice);
2292 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2293 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2294 if (fMaxInitFailed>0)
84fcd93f 2295 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2296 if (fMasterResubmitThreshold>0)
84fcd93f 2297 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2298 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2299 if (fNrunsPerMaster>0)
84fcd93f 2300 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2301 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2302 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2303 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2304 if (fArguments.Length())
84fcd93f 2305 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2306 if (fExecutableArgs.Length())
2307 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2308 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2309 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2310 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2311 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2312 if (fDatasetName)
84fcd93f 2313 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2314 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2315 if (fIncludePath.Data())
84fcd93f 2316 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2317 if (fCloseSE.Length())
84fcd93f 2318 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2319 if (fFriendChainName.Length())
84fcd93f 2320 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2321 if (fPackages && fPackages->GetEntries()) {
16a4353c 2322 TIter next(fPackages);
2323 TObject *obj;
2324 TString list;
2325 while ((obj=next())) list += obj->GetName();
84fcd93f 2326 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2327 }
2328}
2329
c57f56b7 2330//______________________________________________________________________________
2331void AliAnalysisAlien::SetDefaults()
2332{
2333// Set default values for everything. What cannot be filled will be left empty.
2334 if (fGridJDL) delete fGridJDL;
2335 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2336 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2337 fPrice = 1;
2338 fTTL = 30000;
2339 fSplitMaxInputFileNumber = 100;
2340 fMaxInitFailed = 0;
2341 fMasterResubmitThreshold = 0;
bb885a9e 2342 fNtestFiles = 10;
149d288c 2343 fNreplicas = 2;
a8739e8a 2344 fRunRange[0] = 0;
2345 fRunRange[1] = 0;
8e3bd06a 2346 fRunPrefix = "%d";
319593fb 2347 fNrunsPerMaster = 1;
16a4353c 2348 fMaxMergeFiles = 100;
c57f56b7 2349 fRunNumbers = "";
2350 fExecutable = "analysis.sh";
0a1c1f7f 2351 fExecutableCommand = "root -b -q";
c57f56b7 2352 fArguments = "";
631c0b05 2353 fExecutableArgs = "";
c57f56b7 2354 fAnalysisMacro = "myAnalysis.C";
2355 fAnalysisSource = "";
2356 fAdditionalLibs = "";
2357 fSplitMode = "se";
2358 fAPIVersion = "";
2359 fROOTVersion = "";
2360 fAliROOTVersion = "";
2361 fUser = ""; // Your alien user name
2362 fGridWorkingDir = "";
2363 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2364 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2365 fFriendChainName = "";
c57f56b7 2366 fGridOutputDir = "output";
b83f84f0 2367 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2368 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2369 fInputFormat = "xml-single";
2370 fJDLName = "analysis.jdl";
c6cb3634 2371 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2372 fMergeExcludes = "";
0f389141 2373 fMergeViaJDL = 0;
348be253 2374 SetUseCopy(kTRUE);
2375 SetCheckCopy(kTRUE);
149d288c 2376 SetDefaultOutputs(kTRUE);
e1c22e21 2377 fOverwriteMode = 1;
c57f56b7 2378}
2379
0f389141 2380//______________________________________________________________________________
b3e07543 2381Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2382{
b3e07543 2383// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2384 // First check if the result is already in the output directory.
2385 if (FileExists(Form("%s/%s",aliendir,filename))) {
2386 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2387 return kFALSE;
2388 }
b3e07543 2389 // Now check the last stage done.
2390 Int_t stage = 0;
2391 while (1) {
2392 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2393 stage++;
2394 }
2395 // Next stage of merging
2396 stage++;
2397 TString pattern = "*root_archive.zip";
2398 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2399 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2400 if (res) delete res;
2401 // Write standard output to file
2402 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2403 // Count the number of files inside
2404 ifstream ifile;
2405 ifile.open(Form("Stage_%d.xml",stage));
2406 if (!ifile.good()) {
2407 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2408 return kFALSE;
b3e07543 2409 }
2410 TString line;
2411 Int_t nfiles = 0;
2412 while (!ifile.eof()) {
2413 ifile >> line;
2414 if (line.Contains("/event")) nfiles++;
7c2cd90a 2415 }
b3e07543 2416 ifile.close();
2417 if (!nfiles) {
2418 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2419 return kFALSE;
2420 } else {
2421 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2422 }
b3e07543 2423 // Copy the file in the output directory
2424 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2425// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2426 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2427 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2428 // Check if this is the last stage to be done.
2429 Bool_t laststage = (nfiles<nperchunk);
2430 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2431 if (laststage) {
2432 printf("### Submiting final merging stage %d\n", stage);
2433 TString finalJDL = jdl;
2434 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2435 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 2436 Int_t jobId = SubmitSingleJob(query);
2437 if (!jobId) return kFALSE;
2438 } else {
2439 printf("### Submiting merging stage %d\n", stage);
03335324 2440 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 2441 Int_t jobId = SubmitSingleJob(query);
2442 if (!jobId) return kFALSE;
7c2cd90a 2443 }
b3e07543 2444 return kTRUE;
2445}
7c2cd90a 2446
105631b5 2447//______________________________________________________________________________
2448AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2449{
2450// Loat the analysis manager from a file.
2451 TFile *file = TFile::Open(fname);
2452 if (!file) {
2453 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2454 return 0;
2455 }
2456 TIter nextkey(file->GetListOfKeys());
2457 AliAnalysisManager *mgr = 0;
2458 TKey *key;
2459 while ((key=(TKey*)nextkey())) {
2460 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2461 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2462 }
2463 if (!mgr)
2464 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2465 return mgr;
2466}
2467
7c2cd90a 2468//______________________________________________________________________________
2469Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2470{
2471// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2472 if (!gGrid) return 0;
2473 printf("=> %s ------> ",query);
2474 TGridResult *res = gGrid->Command(query);
2475 if (!res) return 0;
2476 TString jobId = res->GetKey(0,"jobId");
2477 delete res;
2478 if (jobId.IsNull()) {
2479 printf("submission failed. Reason:\n");
2480 gGrid->Stdout();
2481 gGrid->Stderr();
2482 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2483 return 0;
2484 }
2485 printf(" Job id: %s\n", jobId.Data());
2486 return atoi(jobId);
2487}
2488
2489//______________________________________________________________________________
b3e07543 2490Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2491{
b3e07543 2492// Merge given output files from basedir. Basedir can be an alien output directory
2493// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2494// files in a group (ignored for xml input). Merging can be done in stages:
2495// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2496// stage=1 : works with an xml of all root_archive.zip in the output directory
2497// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2498 TString outputFile = output;
0f389141 2499 TString command;
a2f5fc01 2500 TString outputChunk;
2501 TString previousChunk = "";
b3e07543 2502 TObjArray *listoffiles = new TObjArray();
2503// listoffiles->SetOwner();
a2f5fc01 2504 Int_t countChunk = 0;
2505 Int_t countZero = nmaxmerge;
0f389141 2506 Bool_t merged = kTRUE;
a2f5fc01 2507 Int_t index = outputFile.Index("@");
2508 if (index > 0) outputFile.Remove(index);
7c2cd90a 2509 TString inputFile = outputFile;
b3e07543 2510 TString sbasedir = basedir;
2511 if (sbasedir.Contains(".xml")) {
2512 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2513 nmaxmerge = 9999999;
2514 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2515 if (!coll) {
2516 ::Error("MergeOutput", "Input XML collection empty.");
2517 return kFALSE;
2518 }
2519 // Iterate grid collection
2520 while (coll->Next()) {
2521 TString fname = gSystem->DirName(coll->GetTURL());
2522 fname += "/";
2523 fname += inputFile;
2524 listoffiles->Add(new TNamed(fname.Data(),""));
2525 }
2526 } else {
2527 command = Form("find %s/ *%s", basedir, inputFile.Data());
2528 printf("command: %s\n", command.Data());
2529 TGridResult *res = gGrid->Command(command);
2530 if (!res) {
2531 ::Error("MergeOutput","No result for the find command\n");
2532 delete listoffiles;
2533 return kFALSE;
2534 }
2535 TIter nextmap(res);
2536 TMap *map = 0;
2537 while ((map=(TMap*)nextmap())) {
2538 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2539 if (!objs || !objs->GetString().Length()) {
2540 // Nothing found - skip this output
2541 delete res;
2542 delete listoffiles;
2543 return kFALSE;
2544 }
2545 listoffiles->Add(new TNamed(objs->GetName(),""));
2546 }
2547 delete res;
2548 }
2549 if (!listoffiles->GetEntries()) {
7c2cd90a 2550 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2551 delete listoffiles;
0f389141 2552 return kFALSE;
2553 }
2554
2555 TFileMerger *fm = 0;
b3e07543 2556 TIter next0(listoffiles);
2557 TObjArray *listoffilestmp = new TObjArray();
2558 listoffilestmp->SetOwner();
2559 TObject *nextfile;
2560 TString snextfile;
2561 // Keep only the files at upper level
2562 Int_t countChar = 0;
2563 while ((nextfile=next0())) {
2564 snextfile = nextfile->GetName();
2565 Int_t crtCount = snextfile.CountChar('/');
2566 if (nextfile == listoffiles->First()) countChar = crtCount;
2567 if (crtCount < countChar) countChar = crtCount;
2568 }
2569 next0.Reset();
2570 while ((nextfile=next0())) {
2571 snextfile = nextfile->GetName();
2572 Int_t crtCount = snextfile.CountChar('/');
2573 if (crtCount > countChar) {
2574 delete nextfile;
2575 continue;
2576 }
2577 listoffilestmp->Add(nextfile);
2578 }
2579 delete listoffiles;
2580 listoffiles = listoffilestmp; // Now contains 'good' files
2581 listoffiles->Print();
2582 TIter next(listoffiles);
7c2cd90a 2583 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2584 outputChunk = outputFile;
2585 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2586 // Check for existent temporary merge files
e1c22e21 2587 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2588 // Preserve old merging functionality for stage 0.
2589 if (stage==0) {
2590 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2591 while (1) {
2592 // Skip as many input files as in a chunk
2ea55496 2593 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2594 nextfile = next();
2595 if (!nextfile) {
2ea55496 2596 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2597 delete listoffiles;
2ea55496 2598 return kFALSE;
2599 }
b3e07543 2600 snextfile = nextfile->GetName();
7c2cd90a 2601 }
2602 outputChunk = outputFile;
2603 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2604 countChunk++;
2605 if (gSystem->AccessPathName(outputChunk)) continue;
2606 // Merged file with chunks up to <countChunk> found
2607 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2608 previousChunk = outputChunk;
2609 break;
2610 }
2611 }
2612 countZero = nmaxmerge;
2613
b3e07543 2614 while ((nextfile=next())) {
2615 snextfile = nextfile->GetName();
2ea55496 2616 // Loop 'find' results and get next LFN
7c2cd90a 2617 if (countZero == nmaxmerge) {
2618 // First file in chunk - create file merger and add previous chunk if any.
2619 fm = new TFileMerger(kFALSE);
2620 fm->SetFastMethod(kTRUE);
2621 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2622 outputChunk = outputFile;
2623 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2624 }
2625 // If last file found, put merged results in the output file
b3e07543 2626 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2627 // Add file to be merged and decrement chunk counter.
b3e07543 2628 fm->AddFile(snextfile);
7c2cd90a 2629 countZero--;
b3e07543 2630 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2631 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2632 // Nothing found - skip this output
2633 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2634 merged = kFALSE;
2635 break;
7c2cd90a 2636 }
2637 fm->OutputFile(outputChunk);
2638 // Merge the outputs, then go to next chunk
2639 if (!fm->Merge()) {
2640 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2641 merged = kFALSE;
2642 break;
7c2cd90a 2643 } else {
2644 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2645 gSystem->Unlink(previousChunk);
2646 }
b3e07543 2647 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2648 countChunk++;
2649 countZero = nmaxmerge;
2650 previousChunk = outputChunk;
0f389141 2651 }
0f389141 2652 }
b3e07543 2653 delete listoffiles;
2654 delete fm;
7c2cd90a 2655 return merged;
2656 }
2657 // Merging stage different than 0.
2658 // Move to the begining of the requested chunk.
7c2cd90a 2659 fm = new TFileMerger(kFALSE);
2660 fm->SetFastMethod(kTRUE);
b3e07543 2661 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2662 delete listoffiles;
7c2cd90a 2663 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2664 // Nothing found - skip this output
2665 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2666 delete fm;
2667 return kFALSE;
2668 }
b3e07543 2669 fm->OutputFile(outputFile);
7c2cd90a 2670 // Merge the outputs
2671 if (!fm->Merge()) {
2672 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2673 delete fm;
2674 return kFALSE;
2675 } else {
b3e07543 2676 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2677 }
7c2cd90a 2678 delete fm;
2679 return kTRUE;
0f389141 2680}
2681
c57f56b7 2682//______________________________________________________________________________
2683Bool_t AliAnalysisAlien::MergeOutputs()
2684{
2685// Merge analysis outputs existing in the AliEn space.
2686 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2687 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2688 if (!Connect()) {
2689 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2690 return kFALSE;
0f389141 2691 }
c9e8f7fd 2692 if (fMergeViaJDL) {
2693 if (!TestBit(AliAnalysisGrid::kMerge)) {
2694 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2695 return kFALSE;
2696 }
2697 if (fProductionMode) {
2698 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2699 return kFALSE;
2700 }
0f389141 2701 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2702 if (!SubmitMerging()) return kFALSE;
0f389141 2703 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2704 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2705 return kFALSE;
c57f56b7 2706 }
2707 // Get the output path
dc030a24 2708 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2709 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2710 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2711 return kFALSE;
2712 }
2713 if (!fOutputFiles.Length()) {
2714 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2715 return kFALSE;
0f389141 2716 }
2717 // Check if fast read option was requested
e1c22e21 2718 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2719 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2720 if (fFastReadOption) {
d3339be3 2721 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2722 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2723 gEnv->SetValue("XNet.ConnectTimeout",50);
2724 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2725 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2726 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2727 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2728 }
e8b839ab 2729 // Make sure we change the temporary directory
2730 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2731 // Set temporary compilation directory to current one
2732 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2733 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2734 TIter next(list);
2735 TObjString *str;
a2f5fc01 2736 TString outputFile;
c57f56b7 2737 Bool_t merged = kTRUE;
2738 while((str=(TObjString*)next())) {
a2f5fc01 2739 outputFile = str->GetString();
2740 Int_t index = outputFile.Index("@");
2741 if (index > 0) outputFile.Remove(index);
2742 TString outputChunk = outputFile;
2743 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2744 // Skip already merged outputs
a2f5fc01 2745 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2746 if (fOverwriteMode) {
a2f5fc01 2747 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2748 gSystem->Unlink(outputFile);
2749 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2750 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2751 outputChunk.Data());
2752 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2753 }
2754 } else {
a2f5fc01 2755 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2756 continue;
2757 }
2758 } else {
a2f5fc01 2759 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2760 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2761 outputChunk.Data());
2762 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2763 }
2764 }
bb885a9e 2765 if (fMergeExcludes.Length() &&
a2f5fc01 2766 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2767 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2768 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2769 if (!merged) {
2770 Error("MergeOutputs", "Terminate() will NOT be executed");
2771 return kFALSE;
ff07ec61 2772 }
2773 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2774 if (fileOpened) fileOpened->Close();
c57f56b7 2775 }
0f389141 2776 return kTRUE;
c57f56b7 2777}
2778
bb885a9e 2779//______________________________________________________________________________
2780void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2781{
2782// Use the output files connected to output containers from the analysis manager
2783// rather than the files defined by SetOutputFiles
2784 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2785 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2786 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2787}
2788
149d288c 2789//______________________________________________________________________________
2790void AliAnalysisAlien::SetOutputFiles(const char *list)
2791{
2792// Manually set the output files list.
2793// Removes duplicates. Not allowed if default outputs are not disabled.
2794 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2795 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2796 return;
2797 }
2798 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2799 fOutputFiles = "";
2800 TString slist = list;
2801 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2802 TObjArray *arr = slist.Tokenize(" ");
2803 TObjString *os;
2804 TIter next(arr);
2805 TString sout;
2806 while ((os=(TObjString*)next())) {
2807 sout = os->GetString();
2808 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2809 if (fOutputFiles.Contains(sout)) continue;
2810 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2811 fOutputFiles += sout;
2812 }
2813 delete arr;
f790bc1b 2814}
149d288c 2815
2816//______________________________________________________________________________
2817void AliAnalysisAlien::SetOutputArchive(const char *list)
2818{
2819// Manually set the output archive list. Free text - you are on your own...
2820// Not allowed if default outputs are not disabled.
2821 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2822 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2823 return;
2824 }
2825 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2826 fOutputArchive = list;
2827}
2828
2829//______________________________________________________________________________
2830void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2831{
2832// Setting a prefered output SE is not allowed anymore.
2833 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2834}
2835
28ce6c9c 2836//______________________________________________________________________________
2837void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
2838{
2839// Set some PROOF special parameter.
2840 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2841 if (pair) {
2842 TObject *old = pair->Key();
2843 TObject *val = pair->Value();
2844 fProofParam.Remove(old);
2845 delete old;
2846 delete val;
2847 }
2848 fProofParam.Add(new TObjString(pname), new TObjString(value));
2849}
2850
2851//______________________________________________________________________________
2852const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
2853{
2854// Returns a special PROOF parameter.
2855 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
2856 if (!pair) return 0;
2857 return pair->Value()->GetName();
2858}
2859
5513444a 2860//______________________________________________________________________________
2861Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2862{
2863// Start remote grid analysis.
3bdcb562 2864 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2865 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2866 if (!mgr || !mgr->IsInitialized()) {
2867 Error("StartAnalysis", "You need an initialized analysis manager for this");
2868 return kFALSE;
2869 }
2870 // Are we in PROOF mode ?
2871 if (mgr->IsProofMode()) {
baa34407 2872 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
2873 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2874 if (fProofCluster.IsNull()) {
2875 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2876 return kFALSE;
2877 }
2878 if (fProofDataSet.IsNull() && !testMode) {
2879 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2880 return kFALSE;
2881 }
2882 // Set the needed environment
2883 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2884 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2885 if (fProofReset && !testMode) {
2886 if (fProofReset==1) {
2887 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2888 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2889 } else {
2890 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2891 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2892 }
2893 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2894 return kFALSE;
2895 }
baa34407 2896
2897 if (!testMode) {
2898 // Check if there is an old active session
2899 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
2900 if (nsessions) {
2901 Error("StartAnalysis","You have to reset your old session first\n");
2902 return kFALSE;
2903 }
53911ca6 2904 }
3bdcb562 2905 // Do we need to change the ROOT version ? The success of this cannot be checked.
2906 if (!fRootVersionForProof.IsNull() && !testMode) {
2907 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2908 fProofCluster.Data(), fRootVersionForProof.Data()));
2909 }
2910 // Connect to PROOF and check the status
2911 Long_t proof = 0;
27734f0e 2912 TString sworkers;
2913 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2914 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2915 if (!testMode) {
27734f0e 2916 if (!sworkers.IsNull())
2917 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2918 else
2919 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2920 } else {
2921 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2922 if (!proof) {
2923 Error("StartAnalysis", "Could not start PROOF in test mode");
2924 return kFALSE;
2925 }
2926 }
2927 if (!proof) {
2928 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2929 return kFALSE;
2930 }
27734f0e 2931 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2932 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 2933 // Set proof special parameters if any
2934 TIter nextpp(&fProofParam);
2935 TObject *proofparam;
2936 while ((proofparam=nextpp())) {
2937 TString svalue = GetProofParameter(proofparam->GetName());
2938 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
2939 }
3bdcb562 2940 // Is dataset existing ?
2941 if (!testMode) {
2942 TString dataset = fProofDataSet;
2943 Int_t index = dataset.Index("#");
2944 if (index>=0) dataset.Remove(index);
830acc4c 2945// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2946// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2947// return kFALSE;
2948// }
2949// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2950 }
2951 // Is ClearPackages() needed ?
2952 if (TestSpecialBit(kClearPackages)) {
2953 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2954 gROOT->ProcessLine("gProof->ClearPackages();");
2955 }
2956 // Is a given aliroot mode requested ?
2957 TList optionsList;
1ac846e9 2958 TString parLibs;
830acc4c 2959 if (!fAliRootMode.IsNull()) {
3bdcb562 2960 TString alirootMode = fAliRootMode;
2961 if (alirootMode == "default") alirootMode = "";
2962 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2963 optionsList.SetOwner();
2964 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2965 // Check the additional libs to be loaded
2966 TString extraLibs;
1ac846e9 2967 Bool_t parMode = kFALSE;
dc1ffbcb 2968 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 2969 // Parse the extra libs for .so
2970 if (fAdditionalLibs.Length()) {
2971 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2972 TIter next(list);
2973 TObjString *str;
1ac846e9 2974 while((str=(TObjString*)next())) {
2975 if (str->GetString().Contains(".so")) {
2976 if (parMode) {
2977 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2978 break;
2979 }
2980 TString stmp = str->GetName();
2981 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2982 stmp.ReplaceAll(".so","");
2983 if (!extraLibs.IsNull()) extraLibs += ":";
2984 extraLibs += stmp;
2985 continue;
2986 }
2987 if (str->GetString().Contains(".par")) {
2988 // The first par file found in the list will not allow any further .so
2989 parMode = kTRUE;
2990 if (!parLibs.IsNull()) parLibs += ":";
2991 parLibs += str->GetName();
2992 continue;
2993 }
3bdcb562 2994 }
2995 if (list) delete list;
2996 }
dc030a24 2997 if (!extraLibs.IsNull()) {
2998 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
2999 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3000 }
830acc4c 3001 // Check extra includes
3002 if (!fIncludePath.IsNull()) {
3003 TString includePath = fIncludePath;
3004 includePath.ReplaceAll(" ",":");
dc030a24 3005 includePath.ReplaceAll("$ALICE_ROOT/","");
3006 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3007 includePath.ReplaceAll("-I","");
dc030a24 3008 includePath.Remove(TString::kTrailing, ':');
830acc4c 3009 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3010 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3011 }
27734f0e 3012 // Check if connection to grid is requested
3013 if (TestSpecialBit(kProofConnectGrid))
3014 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3015 // Enable AliRoot par
3016 if (testMode) {
3017 // Enable proof lite package
3018 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3019 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3020 TNamed *obj = (TNamed*)optionsList.At(i);
3021 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3022 }
3023 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3024 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3025 Info("StartAnalysis", "AliRootProofLite enabled");
3026 } else {
3027 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3028 return kFALSE;
3029 }
3030 } else {
dc030a24 3031 if ( ! fAliROOTVersion.IsNull() ) {
3032 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3033 fAliROOTVersion.Data(), &optionsList))) {
3034 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3035 return kFALSE;
3036 }
3037 }
27734f0e 3038 }
1ac846e9 3039 // Enable first par files from fAdditionalLibs
3040 if (!parLibs.IsNull()) {
3041 TObjArray *list = parLibs.Tokenize(":");
3042 TIter next(list);
3043 TObjString *package;
3044 while((package=(TObjString*)next())) {
40501bb7 3045 TString spkg = package->GetName();
3046 spkg.ReplaceAll(".par", "");
3047 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3048 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3049 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3050 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3051 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3052 return kFALSE;
3053 }
3054 } else {
3055 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3056 return kFALSE;
3057 }
3058 }
3059 if (list) delete list;
3060 }
3bdcb562 3061 } else {
3062 if (fAdditionalLibs.Contains(".so") && !testMode) {
3063 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3064 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3065 return kFALSE;
3066 }
3067 }
3068 // Enable par files if requested
3069 if (fPackages && fPackages->GetEntries()) {
3070 TIter next(fPackages);
3071 TObject *package;
3072 while ((package=next())) {
1ac846e9 3073 // Skip packages already enabled
3074 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3075 TString spkg = package->GetName();
3076 spkg.ReplaceAll(".par", "");
3077 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3078 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3079 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3080 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3081 return kFALSE;
3082 }
3083 } else {
3084 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3085 return kFALSE;
3086 }
3087 }
3088 }
3089 // Do we need to load analysis source files ?
3090 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3091 if (fAnalysisSource.Length()) {
3092 TObjArray *list = fAnalysisSource.Tokenize(" ");
3093 TIter next(list);
3094 TObjString *str;
3095 while((str=(TObjString*)next())) {
3096 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3097 }
3bdcb562 3098 if (list) delete list;
3099 }
830acc4c 3100 if (testMode) {
3101 // Register dataset to proof lite.
3102 if (fFileForTestMode.IsNull()) {
3103 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3104 return kFALSE;
3105 }
3106 if (gSystem->AccessPathName(fFileForTestMode)) {
3107 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3108 return kFALSE;
3109 }
3110 TFileCollection *coll = new TFileCollection();
3111 coll->AddFromFile(fFileForTestMode);
8ddad121 3112 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3113 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3114 }
3bdcb562 3115 return kTRUE;
3116 }
c57f56b7 3117
43da816a 3118 // Check if output files have to be taken from the analysis manager
3119 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3120 // Add output files and AOD files
3121 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3122 // Add extra files registered to the analysis manager
f790bc1b 3123 TString extra = GetListOfFiles("ext");
3124 if (!extra.IsNull()) {
149d288c 3125 extra.ReplaceAll(".root", "*.root");
f790bc1b 3126 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3127 fOutputFiles += extra;
84fcd93f 3128 }
149d288c 3129 // Compose the output archive.
b83f84f0 3130 fOutputArchive = "log_archive.zip:std*@disk=1 ";
3131 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3132 }
f7b1cbc2 3133// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3134 if (TestBit(AliAnalysisGrid::kOffline)) {
3135 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3136 \n there nor any job run. You can revise the JDL and analysis \
3137 \n macro then run the same in \"submit\" mode.");
3138 } else if (TestBit(AliAnalysisGrid::kTest)) {
3139 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3140 \n dataset.");
3141 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3142 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3143 \n space and job submitted.");
3144 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3145 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3146 if (fMergeViaJDL) CheckInputData();
5513444a 3147 return kTRUE;
c57f56b7 3148 } else {
3149 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3150 }
3151
348be253 3152 Print();
c57f56b7 3153 if (!Connect()) {
3154 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3155 return kFALSE;
16a4353c 3156 }
d3b18c4c 3157 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3158 if (!CheckInputData()) {
3159 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3160 return kFALSE;
c57f56b7 3161 }
d3339be3 3162 if (!CreateDataset(fDataPattern)) {
3163 TString serror;
3164 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3165 if (fRunNumbers.Length()) serror = "run numbers";
3166 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3167 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3168 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3169 return kFALSE;
3170 }
21ca8e59 3171 WriteAnalysisFile();
c57f56b7 3172 WriteAnalysisMacro();
3173 WriteExecutable();
3174 WriteValidationScript();
0f389141 3175 if (fMergeViaJDL) {
3176 WriteMergingMacro();
3177 WriteMergeExecutable();
3178 WriteValidationScript(kTRUE);
3179 }
5513444a 3180 if (!CreateJDL()) return kFALSE;
3181 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3182 if (testMode) {
c57f56b7 3183 // Locally testing the analysis
3184 Info("StartAnalysis", "\n_______________________________________________________________________ \
3185 \n Running analysis script in a daughter shell as on a worker node \
3186 \n_______________________________________________________________________");
149d288c 3187 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3188 TIter next(list);
3189 TObjString *str;
a2f5fc01 3190 TString outputFile;
c57f56b7 3191 while((str=(TObjString*)next())) {
a2f5fc01 3192 outputFile = str->GetString();
3193 Int_t index = outputFile.Index("@");
3194 if (index > 0) outputFile.Remove(index);
3195 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3196 }
3197 delete list;
3198 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3199 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3200// gSystem->Exec("cat stdout");
5513444a 3201 return kFALSE;
c57f56b7 3202 }
5513444a 3203 // Check if submitting is managed by LPM manager
a3e84053 3204 if (fProductionMode) {
5513444a 3205 TString prodfile = fJDLName;
3206 prodfile.ReplaceAll(".jdl", ".prod");
3207 WriteProductionFile(prodfile);
3208 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3209 return kFALSE;
3210 }
a8739e8a 3211 // Submit AliEn job(s)
d2a409b2 3212 gGrid->Cd(fGridOutputDir);
a8739e8a 3213 TGridResult *res;
c57f56b7 3214 TString jobID = "";
d2a409b2 3215 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3216 // Submit a given xml or a set of runs
a8739e8a 3217 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3218 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3219 if (res) {
3220 const char *cjobId = res->GetKey(0,"jobId");
3221 if (!cjobId) {
a03be957 3222 gGrid->Stdout();
3223 gGrid->Stderr();
a8739e8a 3224 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3225 return kFALSE;
a8739e8a 3226 } else {
3227 Info("StartAnalysis", "\n_______________________________________________________________________ \
3228 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3229 \n_______________________________________________________________________",
3230 fJDLName.Data(), cjobId);
3231 jobID = cjobId;
3232 }
3233 delete res;
a03be957 3234 } else {
3235 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3236 return kFALSE;
a8739e8a 3237 }
3238 } else {
d2a409b2 3239 // Submit for a range of enumeration of runs.
a03be957 3240 if (!Submit()) return kFALSE;
c57f56b7 3241 }
a8739e8a 3242
c57f56b7 3243 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3244 \n You may exit at any time and terminate the job later using the option <terminate> \
3245 \n ##################################################################################", jobID.Data());
bb885a9e 3246 gSystem->Exec("aliensh");
5513444a 3247 return kTRUE;
c57f56b7 3248}
3249
f790bc1b 3250//______________________________________________________________________________
3251const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3252{
3253// Get a comma-separated list of output files of the requested type.
3254// Type can be (case unsensitive):
3255// aod - list of aod files (std, extensions and filters)
3256// out - list of output files connected to containers (but not aod's or extras)
3257// ext - list of extra files registered to the manager
3258// ter - list of files produced in terminate
3259 static TString files;
3260 files = "";
3261 TString stype = type;
3262 stype.ToLower();
3263 TString aodfiles, extra;
3264 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3265 if (!mgr) {
3266 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3267 return files.Data();
3268 }
3269 if (mgr->GetOutputEventHandler()) {
3270 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3271 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3272 if (!extraaod.IsNull()) {
3273 aodfiles += ",";
3274 aodfiles += extraaod;
3275 }
3276 }
3277 if (stype.Contains("aod")) {
3278 files = aodfiles;
3279 if (stype == "aod") return files.Data();
3280 }
3281 // Add output files that are not in the list of AOD files
3282 TString outputfiles = "";
3283 TIter next(mgr->GetOutputs());
3284 AliAnalysisDataContainer *output;
3285 const char *filename = 0;
3286 while ((output=(AliAnalysisDataContainer*)next())) {
3287 filename = output->GetFileName();
3288 if (!(strcmp(filename, "default"))) continue;
3289 if (outputfiles.Contains(filename)) continue;
3290 if (aodfiles.Contains(filename)) continue;
3291 if (!outputfiles.IsNull()) outputfiles += ",";
3292 outputfiles += filename;
3293 }
3294 if (stype.Contains("out")) {
3295 if (!files.IsNull()) files += ",";
3296 files += outputfiles;
3297 if (stype == "out") return files.Data();
3298 }
3299 // Add extra files registered to the analysis manager
3300 TString sextra;
3301 extra = mgr->GetExtraFiles();
3302 if (!extra.IsNull()) {
3303 extra.Strip();
3304 extra.ReplaceAll(" ", ",");
3305 TObjArray *fextra = extra.Tokenize(",");
3306 TIter nextx(fextra);
3307 TObject *obj;
3308 while ((obj=nextx())) {
3309 if (aodfiles.Contains(obj->GetName())) continue;
3310 if (outputfiles.Contains(obj->GetName())) continue;
3311 if (sextra.Contains(obj->GetName())) continue;
3312 if (!sextra.IsNull()) sextra += ",";
3313 sextra += obj->GetName();
3314 }
3315 delete fextra;
3316 if (stype.Contains("ext")) {
3317 if (!files.IsNull()) files += ",";
3318 files += sextra;
3319 }
3320 }
3321 if (stype == "ext") return files.Data();
3322 TString termfiles;
3323 if (!fTerminateFiles.IsNull()) {
3324 fTerminateFiles.Strip();
3325 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3326 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3327 TIter nextx(fextra);
3328 TObject *obj;
3329 while ((obj=nextx())) {
3330 if (aodfiles.Contains(obj->GetName())) continue;
3331 if (outputfiles.Contains(obj->GetName())) continue;
3332 if (termfiles.Contains(obj->GetName())) continue;
3333 if (sextra.Contains(obj->GetName())) continue;
3334 if (!termfiles.IsNull()) termfiles += ",";
3335 termfiles += obj->GetName();
3336 }
3337 delete fextra;
3338 }
3339 if (stype.Contains("ter")) {
3340 if (!files.IsNull() && !termfiles.IsNull()) {
3341 files += ",";
3342 files += termfiles;
3343 }
3344 }
3345 return files.Data();
3346}
3347
d2a409b2 3348//______________________________________________________________________________
a03be957 3349Bool_t AliAnalysisAlien::Submit()
d2a409b2 3350{
3351// Submit all master jobs.
3352 Int_t nmasterjobs = fInputFiles->GetEntries();
3353 Long_t tshoot = gSystem->Now();
a03be957 3354 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3355 while (fNsubmitted < nmasterjobs) {
3356 Long_t now = gSystem->Now();
3357 if ((now-tshoot)>30000) {
3358 tshoot = now;
a03be957 3359 if (!SubmitNext()) return kFALSE;
d2a409b2 3360 }
3361 }
a03be957 3362 return kTRUE;
d2a409b2 3363}
3364
0f389141 3365//______________________________________________________________________________
a03be957 3366Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3367{
3368// Submit all merging jobs.
dc030a24 3369 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3370 gGrid->Cd(fGridOutputDir);
3371 TString mergeJDLName = fExecutable;
3372 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3373 if (!fInputFiles) {
3374 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3375 return kFALSE;
3376 }
0f389141 3377 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3378 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3379 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3380 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3381 if (fOutputToRunNo) {
3382 // The output directory is the run number
3383 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3384 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3385 } else {
17b4e9a4 3386 if (!fRunNumbers.Length() && !fRunRange[0]) {
3387 // The output directory is the grid outdir
3388 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3389 runOutDir = fGridOutputDir;
3390 } else {
3391 // The output directory is the master number in 3 digits format
3392 printf("### Submitting merging job for master <%03d>\n", i);
3393 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3394 }
7c2cd90a 3395 }
3396 // Check now the number of merging stages.
2b222f7c 3397 TObjArray *list = fOutputFiles.Tokenize(",");
3398 TIter next(list);
3399 TObjString *str;
3400 TString outputFile;
3401 while((str=(TObjString*)next())) {
3402 outputFile = str->GetString();
3403 Int_t index = outputFile.Index("@");
3404 if (index > 0) outputFile.Remove(index);
3405 if (!fMergeExcludes.Contains(outputFile)) break;
3406 }
3407 delete list;
b3e07543 3408 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3409 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3410 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3411 }
a03be957 3412 if (!ntosubmit) return kTRUE;
9f020952 3413 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3414 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3415 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3416 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3417 \n ################################################################################################################");
0f389141 3418 gSystem->Exec("aliensh");
a03be957 3419 return kTRUE;
0f389141 3420}
3421
d2a409b2 3422//______________________________________________________________________________
a03be957 3423Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3424{
08d5b699 3425// Submit next bunch of master jobs if the queue is free. The first master job is
3426// submitted right away, while the next will not be unless the previous was split.
3427// The plugin will not submit new master jobs if there are more that 500 jobs in
3428// waiting phase.
d2a409b2 3429 static Bool_t iscalled = kFALSE;
3430 static Int_t firstmaster = 0;
3431 static Int_t lastmaster = 0;
3432 static Int_t npermaster = 0;
a03be957 3433 if (iscalled) return kTRUE;
d2a409b2 3434 iscalled = kTRUE;
3435 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3436 Int_t ntosubmit = 0;
3437 TGridResult *res;
3438 TString jobID = "";
3bdcb562 3439 Int_t nmasterjobs = fInputFiles->GetEntries();
3440 if (!fNsubmitted) {
3441 ntosubmit = 1;
3442 if (!IsUseSubmitPolicy()) {
90d50a8c 3443 if (nmasterjobs>5)
3bdcb562 3444 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3445 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3446 ntosubmit = nmasterjobs;
3447 }
3448 } else {
d2a409b2 3449 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3450 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3451 // If last master not split, just return
a03be957 3452 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3453 // No more than 100 waiting jobs
08d5b699 3454 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3455 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3456 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3457 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3458 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3459 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3460 }
d2a409b2 3461 for (Int_t i=0; i<ntosubmit; i++) {
3462 // Submit for a range of enumeration of runs.
a03be957 3463 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3464 TString query;
cd11251e 3465 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3466 runOutDir.ReplaceAll(".xml", "");
3467 if (fOutputToRunNo)
3468 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3469 else
3470 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3471 printf("********* %s\n",query.Data());
d2a409b2 3472 res = gGrid->Command(query);
3473 if (res) {
98ca124f 3474 TString cjobId1 = res->GetKey(0,"jobId");
3475 if (!cjobId1.Length()) {
d2a409b2 3476 iscalled = kFALSE;
a03be957 3477 gGrid->Stdout();
3478 gGrid->Stderr();
3479 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3480 return kFALSE;
d2a409b2 3481 } else {
3482 Info("StartAnalysis", "\n_______________________________________________________________________ \
3483 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3484 \n_______________________________________________________________________",
98ca124f 3485 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 3486 jobID += cjobId1;
3487 jobID += " ";
98ca124f 3488 lastmaster = cjobId1.Atoi();
d2a409b2 3489 if (!firstmaster) firstmaster = lastmaster;
3490 fNsubmitted++;
3491 }
3492 delete res;
a03be957 3493 } else {
3494 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3495 return kFALSE;
d2a409b2 3496 }
3497 }
3498 iscalled = kFALSE;
a03be957 3499 return kTRUE;
d2a409b2 3500}
3501
c57f56b7 3502//______________________________________________________________________________
3503void AliAnalysisAlien::WriteAnalysisFile()
3504{
f10e8481 3505// Write current analysis manager into the file <analysisFile>
3506 TString analysisFile = fExecutable;
3507 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3508 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3509 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3510 if (!mgr || !mgr->IsInitialized()) {
3511 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3512 return;
3513 }
3514 // Check analysis type
3515 TObject *handler;
3516 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3517 handler = (TObject*)mgr->GetInputEventHandler();
3518 if (handler) {
65eb22f8 3519 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3520 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3521 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3522 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3523 } else {
3524 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3525 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3526 }
c57f56b7 3527 }
3528 TDirectory *cdir = gDirectory;
f10e8481 3529 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3530 if (file) {
e16a394c 3531 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3532 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3533 // Unless merging makes no sense
3534 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3535 mgr->Write();
3536 delete file;
fe2d7fc2 3537 // Enable termination for local jobs
3538 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3539 }
3540 if (cdir) cdir->cd();
f10e8481 3541 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3542 }
3543 Bool_t copy = kTRUE;
d3b18c4c 3544 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3545 if (copy) {
3546 CdWork();
3547 TString workdir = gGrid->GetHomeDirectory();
3548 workdir += fGridWorkingDir;
d3b18c4c 3549 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3550 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3551 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3552 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3553 }
3554}
3555
3556//______________________________________________________________________________
3557void AliAnalysisAlien::WriteAnalysisMacro()
3558{
3559// Write the analysis macro that will steer the analysis in grid mode.
3560 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3561 ofstream out;
3562 out.open(fAnalysisMacro.Data(), ios::out);
3563 if (!out.good()) {
3564 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3565 return;
3566 }
5b273635 3567 Bool_t hasSTEERBase = kFALSE;
3568 Bool_t hasESD = kFALSE;
3569 Bool_t hasAOD = kFALSE;
3570 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3571 Bool_t hasOADB = kFALSE;
5b273635 3572 Bool_t hasANALYSISalice = kFALSE;
3573 Bool_t hasCORRFW = kFALSE;
c57f56b7 3574 TString func = fAnalysisMacro;
3575 TString type = "ESD";
3576 TString comment = "// Analysis using ";
c85cfc0f 3577 if (IsUseMCchain()) {
3578 type = "MC";
3579 comment += "MC";
3580 } else {
3581 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3582 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3583 type = "AOD";
3584 comment += "AOD";
3585 }
3586 }
0df6ccf2 3587 if (type!="AOD" && fFriendChainName!="") {
3588 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3589 return;
3590 }
c57f56b7 3591 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3592 else comment += " data";
3593 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3594 func.ReplaceAll(".C", "");
3595 out << "void " << func.Data() << "()" << endl;
3596 out << "{" << endl;
3597 out << comment.Data() << endl;
3598 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3599 out << " TStopwatch timer;" << endl;
3600 out << " timer.Start();" << endl << endl;
13f0c00b 3601 // Change temp directory to current one
3602 out << "// Set temporary merging directory to current one" << endl;
3603 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3604 out << "// Set temporary compilation directory to current one" << endl;
3605 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3606 // Reset existing include path
3607 out << "// Reset existing include path and add current directory first in the search" << endl;
3608 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3609 if (!fExecutableCommand.Contains("aliroot")) {
3610 out << "// load base root libraries" << endl;
3611 out << " gSystem->Load(\"libTree\");" << endl;
3612 out << " gSystem->Load(\"libGeom\");" << endl;
3613 out << " gSystem->Load(\"libVMC\");" << endl;
3614 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3615 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3616 }
d5c6455a 3617 if (fAdditionalRootLibs.Length()) {
47a4137d 3618 // in principle libtree /lib geom libvmc etc. can go into this list, too
3619 out << "// Add aditional libraries" << endl;
3620 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3621 TIter next(list);
3622 TObjString *str;
3623 while((str=(TObjString*)next())) {
3624 if (str->GetString().Contains(".so"))
3625 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3626 }
47a4137d 3627 if (list) delete list;
d5c6455a 3628 }
57377eb5 3629 out << "// Load analysis framework libraries" << endl;
652561d5 3630 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3631 if (!fPackages) {
f790bc1b 3632 if (!fExecutableCommand.Contains("aliroot")) {
3633 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3634 out << " gSystem->Load(\"libESD\");" << endl;
3635 out << " gSystem->Load(\"libAOD\");" << endl;
3636 }
4e5c5506 3637 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3638 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3639 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3640 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3641 } else {
4e5c5506 3642 TIter next(fPackages);
3643 TObject *obj;
57377eb5 3644 TString pkgname;
57377eb5 3645 while ((obj=next())) {
3646 pkgname = obj->GetName();
4478e6f1 3647 if (pkgname == "STEERBase" ||
3648 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3649 if (pkgname == "ESD" ||
3650 pkgname == "ESD.par") hasESD = kTRUE;
3651 if (pkgname == "AOD" ||
3652 pkgname == "AOD.par") hasAOD = kTRUE;
3653 if (pkgname == "ANALYSIS" ||
3654 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3655 if (pkgname == "OADB" ||
3656 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 3657 if (pkgname == "ANALYSISalice" ||
3658 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3659 if (pkgname == "CORRFW" ||
3660 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3661 }
3662 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3663 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3664 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3665 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3666 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3667 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3668 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3669 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3670 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 3671 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
3672 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 3673 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3674 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3675 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3676 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3677 out << "// Compile other par packages" << endl;
3678 next.Reset();
fcc9bb6f 3679 while ((obj=next())) {
3680 pkgname = obj->GetName();
4478e6f1 3681 if (pkgname == "STEERBase" ||
3682 pkgname == "STEERBase.par" ||
3683 pkgname == "ESD" ||
3684 pkgname == "ESD.par" ||
3685 pkgname == "AOD" ||
3686 pkgname == "AOD.par" ||
3687 pkgname == "ANALYSIS" ||
3688 pkgname == "ANALYSIS.par" ||
7ea40ac8 3689 pkgname == "OADB" ||
3690 pkgname == "OADB.par" ||
4478e6f1 3691 pkgname == "ANALYSISalice" ||
3692 pkgname == "ANALYSISalice.par" ||
3693 pkgname == "CORRFW" ||
3694 pkgname == "CORRFW.par") continue;
5b273635 3695 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3696 }
4e5c5506 3697 }
b3e07543 3698 out << "// include path" << endl;
3699 // Get the include path from the interpreter and remove entries pointing to AliRoot
3700 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3701 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3702 out << " TIter nextpath(listpaths);" << endl;
3703 out << " TObjString *pname;" << endl;
3704 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3705 out << " TString current = pname->GetName();" << endl;
3706 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3707 out << " gSystem->AddIncludePath(current);" << endl;
3708 out << " }" << endl;
3709 out << " if (listpaths) delete listpaths;" << endl;
3710 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3711 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3712 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3713 if (fAdditionalLibs.Length()) {
3714 out << "// Add aditional AliRoot libraries" << endl;
3715 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3716 TIter next(list);
3717 TObjString *str;
3718 while((str=(TObjString*)next())) {
3719 if (str->GetString().Contains(".so"))
3720 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3721 if (str->GetString().Contains(".par"))
3722 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3723 }
3724 if (list) delete list;
3725 }
3726 out << endl;
c57f56b7 3727 out << "// analysis source to be compiled at runtime (if any)" << endl;
3728 if (fAnalysisSource.Length()) {
3729 TObjArray *list = fAnalysisSource.Tokenize(" ");
3730 TIter next(list);
3731 TObjString *str;
3732 while((str=(TObjString*)next())) {
3733 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3734 }
3735 if (list) delete list;
3736 }
3737 out << endl;
b3e07543 3738// out << " printf(\"Currently load libraries:\\n\");" << endl;
3739// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3740 if (fFastReadOption) {
d3339be3 3741 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3742 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3743 out << "// fast xrootd reading enabled" << endl;
3744 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3745 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3746 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3747 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3748 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3749 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 3750 }
3751 if (!IsLocalTest()) {
3752 out << "// connect to AliEn and make the chain" << endl;
3753 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0f389141 3754 }
c57f56b7 3755 out << "// read the analysis manager from file" << endl;
f10e8481 3756 TString analysisFile = fExecutable;
3757 analysisFile.ReplaceAll(".sh", ".root");
105631b5 3758 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
3759 << analysisFile << "\");" << endl;
3760 out << " if (!mgr) return;" << endl;
21ca8e59 3761 if (IsLocalTest()) {
3762 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
3763 out << " plugin->SetRunMode(\"test\");" << endl;
3764 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
3765 out << " mgr->SetGridHandler(plugin);" << endl;
3766 out << " mgr->SetDebugLevel(10);" << endl;
3767 out << " mgr->SetNSysInfo(100);" << endl;
3768 }
c57f56b7 3769 out << " mgr->PrintStatus();" << endl;
52b6a92b 3770 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3771 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3772 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3773 } else {
9a8288ff 3774 if (TestBit(AliAnalysisGrid::kTest))
3775 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3776 else
3777 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3778 }
3779 }
21ca8e59 3780 if (!IsLocalTest()) {
242accb2 3781 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 3782 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3783 } else {
3784 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 3785 }
f7498086 3786 out << " timer.Stop();" << endl;
3787 out << " timer.Print();" << endl;
c57f56b7 3788 out << "}" << endl << endl;
21ca8e59 3789 if (!IsLocalTest()) {
fcc9bb6f 3790 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3791 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3792 out << "{" << endl;
3793 out << "// Create a chain using url's from xml file" << endl;
242accb2 3794 out << " TString filename;" << endl;
3795 out << " Int_t run = 0;" << endl;
c85cfc0f 3796 if (IsUseMCchain()) {
3797 out << " TString treename = \"TE\";" << endl;
3798 } else {
3799 out << " TString treename = type;" << endl;
3800 out << " treename.ToLower();" << endl;
3801 out << " treename += \"Tree\";" << endl;
3802 }
e02fee64 3803 out << " printf(\"***************************************\\n\");" << endl;
3804 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3805 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3806 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3807 out << " if (!coll) {" << endl;
3808 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3809 out << " return NULL;" << endl;
3810 out << " }" << endl;
242accb2 3811 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3812 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3813 if(fFriendChainName!="") {
3814 out << " TChain *chainFriend = new TChain(treename);" << endl;
3815 }
c57f56b7 3816 out << " coll->Reset();" << endl;
0df6ccf2 3817 out << " while (coll->Next()) {" << endl;
242accb2 3818 out << " filename = coll->GetTURL("");" << endl;
3819 out << " if (mgr) {" << endl;
3820 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3821 out << " if (nrun && nrun != run) {" << endl;
3822 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3823 out << " mgr->SetRunFromPath(nrun);" << endl;
3824 out << " run = nrun;" << endl;
3825 out << " }" << endl;
3826 out << " }" << endl;
3827 out << " chain->Add(filename);" << endl;
0df6ccf2 3828 if(fFriendChainName!="") {
3829 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3830 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3831 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3832 out << " chainFriend->Add(fileFriend.Data());" << endl;
3833 }
3834 out << " }" << endl;
c57f56b7 3835 out << " if (!chain->GetNtrees()) {" << endl;
3836 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3837 out << " return NULL;" << endl;
3838 out << " }" << endl;
0df6ccf2 3839 if(fFriendChainName!="") {
3840 out << " chain->AddFriend(chainFriend);" << endl;
3841 }
c57f56b7 3842 out << " return chain;" << endl;
fcc9bb6f 3843 out << "}" << endl << endl;
c57f56b7 3844 }
5b273635 3845 if (hasANALYSISalice) {
3846 out <<"//________________________________________________________________________________" << endl;
3847 out << "Bool_t SetupPar(const char *package) {" << endl;
3848 out << "// Compile the package and set it up." << endl;
3849 out << " TString pkgdir = package;" << endl;
3850 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3851 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3852 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3853 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3854 out << " // Check for BUILD.sh and execute" << endl;
3855 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3856 out << " printf(\"*******************************\\n\");" << endl;
3857 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3858 out << " printf(\"*******************************\\n\");" << endl;
3859 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3860 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3861 out << " gSystem->ChangeDirectory(cdir);" << endl;
3862 out << " return kFALSE;" << endl;
3863 out << " }" << endl;
3864 out << " } else {" << endl;
3865 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3866 out << " gSystem->ChangeDirectory(cdir);" << endl;
3867 out << " return kFALSE;" << endl;
3868 out << " }" << endl;
3869 out << " // Check for SETUP.C and execute" << endl;
3870 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3871 out << " printf(\"*******************************\\n\");" << endl;
3872 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3873 out << " printf(\"*******************************\\n\");" << endl;
3874 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3875 out << " } else {" << endl;
3876 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3877 out << " gSystem->ChangeDirectory(cdir);" << endl;
3878 out << " return kFALSE;" << endl;
3879 out << " }" << endl;
3880 out << " // Restore original workdir" << endl;
3881 out << " gSystem->ChangeDirectory(cdir);" << endl;
3882 out << " return kTRUE;" << endl;
3883 out << "}" << endl;
3884 }
c57f56b7 3885 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3886 }
3887 Bool_t copy = kTRUE;
d3b18c4c 3888 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3889 if (copy) {
3890 CdWork();
3891 TString workdir = gGrid->GetHomeDirectory();
3892 workdir += fGridWorkingDir;
3893 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 3894 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 3895// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3896 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
3897 Form("alien://%s/%s", workdir.Data(),
3898 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 3899 }
3900}
3901
0f389141 3902//______________________________________________________________________________
3903void AliAnalysisAlien::WriteMergingMacro()
3904{
3905// Write a macro to merge the outputs per master job.
3906 if (!fMergeViaJDL) return;
3907 if (!fOutputFiles.Length()) {
3908 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3909 return;
3910 }
3911 TString mergingMacro = fExecutable;
3912 mergingMacro.ReplaceAll(".sh","_merge.C");
dc030a24 3913 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3914 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3915 ofstream out;
3916 out.open(mergingMacro.Data(), ios::out);
3917 if (!out.good()) {
3918 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3919 return;
3920 }
5b273635 3921 Bool_t hasSTEERBase = kFALSE;
3922 Bool_t hasESD = kFALSE;
3923 Bool_t hasAOD = kFALSE;
3924 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3925 Bool_t hasOADB = kFALSE;
5b273635 3926 Bool_t hasANALYSISalice = kFALSE;
3927 Bool_t hasCORRFW = kFALSE;
0f389141 3928 TString func = mergingMacro;
3929 TString comment;
3930 func.ReplaceAll(".C", "");
f47d5cb4 3931 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3932 out << "{" << endl;
3933 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3934 out << " TStopwatch timer;" << endl;
3935 out << " timer.Start();" << endl << endl;
b3e07543 3936 // Reset existing include path
3937 out << "// Reset existing include path and add current directory first in the search" << endl;
3938 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3939 if (!fExecutableCommand.Contains("aliroot")) {
3940 out << "// load base root libraries" << endl;
3941 out << " gSystem->Load(\"libTree\");" << endl;
3942 out << " gSystem->Load(\"libGeom\");" << endl;
3943 out << " gSystem->Load(\"libVMC\");" << endl;
3944 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3945 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3946 }
0f389141 3947 if (fAdditionalRootLibs.Length()) {
3948 // in principle libtree /lib geom libvmc etc. can go into this list, too
3949 out << "// Add aditional libraries" << endl;
3950 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3951 TIter next(list);
3952 TObjString *str;
3953 while((str=(TObjString*)next())) {
3954 if (str->GetString().Contains(".so"))
3955 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3956 }
3957 if (list) delete list;
3958 }
0f389141 3959 out << "// Load analysis framework libraries" << endl;
3960 if (!fPackages) {
7c2cd90a 3961 if (!fExecutableCommand.Contains("aliroot")) {
3962 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3963 out << " gSystem->Load(\"libESD\");" << endl;
3964 out << " gSystem->Load(\"libAOD\");" << endl;
3965 }
0f389141 3966 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3967 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 3968 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3969 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3970 } else {
3971 TIter next(fPackages);
3972 TObject *obj;
3973 TString pkgname;
5b273635 3974 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3975 while ((obj=next())) {
3976 pkgname = obj->GetName();
3977 if (pkgname == "STEERBase" ||
3978 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3979 if (pkgname == "ESD" ||
3980 pkgname == "ESD.par") hasESD = kTRUE;
3981 if (pkgname == "AOD" ||
3982 pkgname == "AOD.par") hasAOD = kTRUE;
3983 if (pkgname == "ANALYSIS" ||
3984 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 3985 if (pkgname == "OADB" ||
3986 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 3987 if (pkgname == "ANALYSISalice" ||
3988 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3989 if (pkgname == "CORRFW" ||
3990 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3991 }
5b273635 3992 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3993 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3994 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3995 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3996 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3997 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3998 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 3999 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4000 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4001 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4002 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4003 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4004 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4005 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4006 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4007 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4008 out << "// Compile other par packages" << endl;
4009 next.Reset();
4010 while ((obj=next())) {
4011 pkgname = obj->GetName();
4012 if (pkgname == "STEERBase" ||
4013 pkgname == "STEERBase.par" ||
4014 pkgname == "ESD" ||
4015 pkgname == "ESD.par" ||
4016 pkgname == "AOD" ||
4017 pkgname == "AOD.par" ||
4018 pkgname == "ANALYSIS" ||
4019 pkgname == "ANALYSIS.par" ||
7ea40ac8 4020 pkgname == "OADB" ||
4021 pkgname == "OADB.par" ||
0f389141 4022 pkgname == "ANALYSISalice" ||
4023 pkgname == "ANALYSISalice.par" ||
4024 pkgname == "CORRFW" ||
4025 pkgname == "CORRFW.par") continue;
5b273635 4026 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4027 }
4028 }
b3e07543 4029 out << "// include path" << endl;
4030 // Get the include path from the interpreter and remove entries pointing to AliRoot
4031 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4032 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4033 out << " TIter nextpath(listpaths);" << endl;
4034 out << " TObjString *pname;" << endl;
4035 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4036 out << " TString current = pname->GetName();" << endl;
4037 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4038 out << " gSystem->AddIncludePath(current);" << endl;
4039 out << " }" << endl;
4040 out << " if (listpaths) delete listpaths;" << endl;
4041 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4042 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4043 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4044 if (fAdditionalLibs.Length()) {
4045 out << "// Add aditional AliRoot libraries" << endl;
4046 TObjArray *list = fAdditionalLibs.Tokenize(" ");
4047 TIter next(list);
4048 TObjString *str;
4049 while((str=(TObjString*)next())) {
4050 if (str->GetString().Contains(".so"))
4051 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4052 }
4053 if (list) delete list;
4054 }
4055 out << endl;
4056 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4057 if (fAnalysisSource.Length()) {
4058 TObjArray *list = fAnalysisSource.Tokenize(" ");
4059 TIter next(list);
4060 TObjString *str;
4061 while((str=(TObjString*)next())) {
4062 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4063 }
4064 if (list) delete list;
4065 }
149d288c 4066 out << endl;
4067
0f389141 4068 if (fFastReadOption) {
4069 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4070 out << "// fast xrootd reading enabled" << endl;
4071 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4072 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4073 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4074 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4075 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4076 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4077 }
4078 // Change temp directory to current one
4079 out << "// Set temporary merging directory to current one" << endl;
4080 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4081 out << "// Set temporary compilation directory to current one" << endl;
4082 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 4083 out << "// Connect to AliEn" << endl;
4084 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 4085 out << " TString outputDir = dir;" << endl;
f790bc1b 4086 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 4087 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 4088 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4089 out << " TIter *iter = new TIter(list);" << endl;
4090 out << " TObjString *str;" << endl;
a2f5fc01 4091 out << " TString outputFile;" << endl;
0f389141 4092 out << " Bool_t merged = kTRUE;" << endl;
4093 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4094 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4095 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4096 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4097 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4098 out << " // Skip already merged outputs" << endl;
a2f5fc01 4099 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4100 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 4101 out << " continue;" << endl;
4102 out << " }" << endl;
a2f5fc01 4103 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4104 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4105 out << " if (!merged) {" << endl;
a2f5fc01 4106 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4107 out << " return;" << endl;
0f389141 4108 out << " }" << endl;
4109 out << " }" << endl;
7c2cd90a 4110 out << " // all outputs merged, validate" << endl;
4111 out << " ofstream out;" << endl;
4112 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4113 out << " out.close();" << endl;
4114 out << " // read the analysis manager from file" << endl;
f866cba5 4115 TString analysisFile = fExecutable;
4116 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 4117 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
105631b5 4118 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4119 << analysisFile << "\");" << endl;
4120 out << " if (!mgr) return;" << endl;
5c4250fc 4121 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4122 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4123 out << " mgr->PrintStatus();" << endl;
4124 if (AliAnalysisManager::GetAnalysisManager()) {
4125 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4126 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4127 } else {
9a8288ff 4128 if (TestBit(AliAnalysisGrid::kTest))
4129 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4130 else
4131 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4132 }
4133 }
9a8288ff 4134 out << " TTree *tree = NULL;" << endl;
4135 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4136 out << "}" << endl << endl;
5b273635 4137 if (hasANALYSISalice) {
4138 out <<"//________________________________________________________________________________" << endl;
4139 out << "Bool_t SetupPar(const char *package) {" << endl;
4140 out << "// Compile the package and set it up." << endl;
4141 out << " TString pkgdir = package;" << endl;
4142 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4143 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4144 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4145 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4146 out << " // Check for BUILD.sh and execute" << endl;
4147 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4148 out << " printf(\"*******************************\\n\");" << endl;
4149 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4150 out << " printf(\"*******************************\\n\");" << endl;
4151 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4152 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4153 out << " gSystem->ChangeDirectory(cdir);" << endl;
4154 out << " return kFALSE;" << endl;
4155 out << " }" << endl;
4156 out << " } else {" << endl;
4157 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4158 out << " gSystem->ChangeDirectory(cdir);" << endl;
4159 out << " return kFALSE;" << endl;
4160 out << " }" << endl;
4161 out << " // Check for SETUP.C and execute" << endl;
4162 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4163 out << " printf(\"*******************************\\n\");" << endl;
4164 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4165 out << " printf(\"*******************************\\n\");" << endl;
4166 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4167 out << " } else {" << endl;
4168 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4169 out << " gSystem->ChangeDirectory(cdir);" << endl;
4170 out << " return kFALSE;" << endl;
4171 out << " }" << endl;
4172 out << " // Restore original workdir" << endl;
4173 out << " gSystem->ChangeDirectory(cdir);" << endl;
4174 out << " return kTRUE;" << endl;
4175 out << "}" << endl;
4176 }
0f389141 4177 }
4178 Bool_t copy = kTRUE;
d3b18c4c 4179 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4180 if (copy) {
4181 CdWork();
4182 TString workdir = gGrid->GetHomeDirectory();
4183 workdir += fGridWorkingDir;
4184 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4185 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4186// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4187 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4188 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4189 }
4190}
4191
4192//______________________________________________________________________________
4193Bool_t AliAnalysisAlien::SetupPar(const char *package)
4194{
205b201f 4195// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4196// Note that for loading the compiled library. The current directory should have precedence in
4197// LD_LIBRARY_PATH
4198 TString pkgdir = package;
4199 pkgdir.ReplaceAll(".par","");
b50a41c3 4200 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4201 TString cdir = gSystem->WorkingDirectory();
4202 gSystem->ChangeDirectory(pkgdir);
4203 // Check for BUILD.sh and execute
4204 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4205 printf("**************************************************\n");
4206 printf("*** Building PAR archive %s\n", package);
4207 printf("**************************************************\n");
4208 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4209 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4210 gSystem->ChangeDirectory(cdir);
4211 return kFALSE;
4212 }
4213 } else {
4214 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4215 gSystem->ChangeDirectory(cdir);
4216 return kFALSE;
4217 }
4218 // Check for SETUP.C and execute
4219 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4220 printf("**************************************************\n");
4221 printf("*** Setup PAR archive %s\n", package);
4222 printf("**************************************************\n");
4223 gROOT->Macro("PROOF-INF/SETUP.C");
4224 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4225 } else {
4226 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4227 gSystem->ChangeDirectory(cdir);
4228 return kFALSE;
4229 }
4230 // Restore original workdir
4231 gSystem->ChangeDirectory(cdir);
4232 return kTRUE;
4233}
4234
c57f56b7 4235//______________________________________________________________________________
4236void AliAnalysisAlien::WriteExecutable()
4237{
4238// Generate the alien executable script.
4239 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4240 ofstream out;
4241 out.open(fExecutable.Data(), ios::out);
4242 if (out.bad()) {
5513444a 4243 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4244 return;
4245 }
4246 out << "#!/bin/bash" << endl;
2a4d5166 4247 // Make sure we can properly compile par files
4248 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4249 out << "echo \"=========================================\"" << endl;
4250 out << "echo \"############## PATH : ##############\"" << endl;
4251 out << "echo $PATH" << endl;
4252 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4253 out << "echo $LD_LIBRARY_PATH" << endl;
4254 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4255 out << "echo $ROOTSYS" << endl;
4256 out << "echo \"############## which root : ##############\"" << endl;
4257 out << "which root" << endl;
4258 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4259 out << "echo $ALICE_ROOT" << endl;
4260 out << "echo \"############## which aliroot : ##############\"" << endl;
4261 out << "which aliroot" << endl;
9c5ddadc 4262 out << "echo \"############## system limits : ##############\"" << endl;
4263 out << "ulimit -a" << endl;
4264 out << "echo \"############## memory : ##############\"" << endl;
4265 out << "free -m" << endl;
c57f56b7 4266 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4267 out << fExecutableCommand << " ";
631c0b05 4268 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 4269 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
4270 out << "echo \"############## memory after: ##############\"" << endl;
4271 out << "free -m" << endl;
c57f56b7 4272 }
4273 Bool_t copy = kTRUE;
d3b18c4c 4274 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4275 if (copy) {
4276 CdWork();
4277 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4278 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4279 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4280 workdir += fGridWorkingDir;
4281 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4282 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4283 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4284// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4285 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4286 executable.Data())) Fatal("","Terminating");
c57f56b7 4287 }
4288}
4289
0f389141 4290//______________________________________________________________________________
4291void AliAnalysisAlien::WriteMergeExecutable()
4292{
4293// Generate the alien executable script for the merging job.
4294 if (!fMergeViaJDL) return;
4295 TString mergeExec = fExecutable;
4296 mergeExec.ReplaceAll(".sh", "_merge.sh");
4297 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4298 ofstream out;
4299 out.open(mergeExec.Data(), ios::out);
4300 if (out.bad()) {
4301 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4302 return;
4303 }
4304 out << "#!/bin/bash" << endl;
2a4d5166 4305 // Make sure we can properly compile par files
4306 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4307 out << "echo \"=========================================\"" << endl;
4308 out << "echo \"############## PATH : ##############\"" << endl;
4309 out << "echo $PATH" << endl;
4310 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4311 out << "echo $LD_LIBRARY_PATH" << endl;
4312 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4313 out << "echo $ROOTSYS" << endl;
4314 out << "echo \"############## which root : ##############\"" << endl;
4315 out << "which root" << endl;
4316 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4317 out << "echo $ALICE_ROOT" << endl;
4318 out << "echo \"############## which aliroot : ##############\"" << endl;
4319 out << "which aliroot" << endl;
4320 out << "echo \"############## system limits : ##############\"" << endl;
4321 out << "ulimit -a" << endl;
4322 out << "echo \"############## memory : ##############\"" << endl;
4323 out << "free -m" << endl;
4324 out << "echo \"=========================================\"" << endl << endl;
0f389141 4325 TString mergeMacro = fExecutable;
4326 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4327 if (IsOneStageMerging())
4328 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4329 else
f47d5cb4 4330 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4331 out << fExecutableCommand << " " << "$ARG" << endl;
4332 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4333 out << "echo \"############## memory after: ##############\"" << endl;
4334 out << "free -m" << endl;
0f389141 4335 }
4336 Bool_t copy = kTRUE;
d3b18c4c 4337 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4338 if (copy) {
4339 CdWork();
4340 TString workdir = gGrid->GetHomeDirectory();
4341 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4342 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4343 workdir += fGridWorkingDir;
4344 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4345 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4346 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4347// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4348 if (!copyLocal2Alien("WriteMergeExecutable",
4349 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4350 }
4351}
4352
c57f56b7 4353//______________________________________________________________________________
5513444a 4354void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4355{
4356// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4357// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4358// Next lines: full_path_to_dataset XXX (XXX is a string)
4359// To submit, one has to: submit jdl XXX for all lines
4360 ofstream out;
4361 out.open(filename, ios::out);
4362 if (out.bad()) {
4363 Error("WriteProductionFile", "Bad file name: %s", filename);
4364 return;
4365 }
d3b18c4c 4366 TString workdir;
4367 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4368 workdir = gGrid->GetHomeDirectory();
5513444a 4369 workdir += fGridWorkingDir;
f5e8c702 4370 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4371 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4372 out << locjdl << " " << njobspermaster << endl;
5513444a 4373 Int_t nmasterjobs = fInputFiles->GetEntries();
4374 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4375 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4376 runOutDir.ReplaceAll(".xml", "");
4377 if (fOutputToRunNo)
4378 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4379 else
4380 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4381 }
d3b18c4c 4382 if (gGrid) {
4383 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4384 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4385// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4386 if (!copyLocal2Alien("WriteProductionFile", filename,
4387 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4388 }
5513444a 4389}
4390
4391//______________________________________________________________________________
0f389141 4392void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4393{
4394// Generate the alien validation script.
4395 // Generate the validation script
4396 TObjString *os;
d3b18c4c 4397 if (fValidationScript.IsNull()) {
4398 fValidationScript = fExecutable;
4399 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4400 }
4401 TString validationScript = fValidationScript;
4402 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4403 if (!Connect()) {
4404 Error("WriteValidationScript", "Alien connection required");
4405 return;
4406 }
d3b18c4c 4407 if (!fTerminateFiles.IsNull()) {
4408 fTerminateFiles.Strip();
4409 fTerminateFiles.ReplaceAll(" ",",");
4410 }
a2f5fc01 4411 TString outStream = "";
4412 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4413 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4414 ofstream out;
0d5d317c 4415 out.open(validationScript, ios::out);
c57f56b7 4416 out << "#!/bin/bash" << endl;
4417 out << "##################################################" << endl;
4418 out << "validateout=`dirname $0`" << endl;
4419 out << "validatetime=`date`" << endl;
4420 out << "validated=\"0\";" << endl;
4421 out << "error=0" << endl;
4422 out << "if [ -z $validateout ]" << endl;
4423 out << "then" << endl;
4424 out << " validateout=\".\"" << endl;
4425 out << "fi" << endl << endl;
4426 out << "cd $validateout;" << endl;
4427 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4428 out << "echo \"*******************************************************\"" << outStream << endl;
4429 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4430 out << "" << endl;
a2f5fc01 4431 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4432 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4433 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4434 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4435 out << "ls -la ./" << outStream << endl;
4436 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4437 out << "##################################################" << endl;
ebec370a 4438 out << "" << endl;
4439
4440 out << "if [ ! -f stderr ] ; then" << endl;
4441 out << " error=1" << endl;
a2f5fc01 4442 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4443 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4444 out << "fi" << endl;
4445
b34c9f51 4446 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4447 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4448 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4449 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4450 out << "" << endl;
4451
ebec370a 4452 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4453 out << " error=1" << endl;
a2f5fc01 4454 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4455 out << " echo \"$parArch\" " << outStream << endl;
4456 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4457 out << "fi" << endl;
4458
4459 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4460 out << " error=1" << endl;
a2f5fc01 4461 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4462 out << " echo \"$segViol\" " << outStream << endl;
4463 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4464 out << "fi" << endl;
4465
4466 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4467 out << " error=1" << endl;
a2f5fc01 4468 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4469 out << " echo \"$segFault\" " << outStream << endl;
4470 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4471 out << "fi" << endl;
4472
b34c9f51 4473 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4474 out << " error=1" << endl;
a2f5fc01 4475 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4476 out << " echo \"$glibcErr\" " << outStream << endl;
4477 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4478 out << "fi" << endl;
4479
ebec370a 4480 // Part dedicated to the specific analyses running into the train
4481
d3b18c4c 4482 TString outputFiles = fOutputFiles;
4483 if (merge && !fTerminateFiles.IsNull()) {
4484 outputFiles += ",";
4485 outputFiles += fTerminateFiles;
4486 }
4487 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4488 TIter next1(arr);
a2f5fc01 4489 TString outputFile;
bbd88db8 4490 while (!merge && (os=(TObjString*)next1())) {
4491 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4492 outputFile = os->GetString();
4493 Int_t index = outputFile.Index("@");
4494 if (index > 0) outputFile.Remove(index);
bbd88db8 4495 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4496 if (outputFile.Contains("*")) continue;
4497 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4498 out << " error=1" << endl;
7c2cd90a 4499 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4500 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4501 out << "fi" << endl;
4502 }
4503 delete arr;
7c2cd90a 4504 out << "if ! [ -f outputs_valid ] ; then" << endl;
4505 out << " error=1" << endl;
4506 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4507 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4508 out << "fi" << endl;
923e2ca5 4509
c57f56b7 4510 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4511 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4512 if (!IsKeepLogs()) {
4513 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4514 outStream = "";
149d288c 4515 out << " rm -f std*" << endl;
4516 }
c57f56b7 4517 out << "fi" << endl;
4518
a2f5fc01 4519 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4520 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4521 out << "cd -" << endl;
4522 out << "exit $error" << endl;
4523 }
4524 Bool_t copy = kTRUE;
d3b18c4c 4525 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4526 if (copy) {
4527 CdWork();
4528 TString workdir = gGrid->GetHomeDirectory();
4529 workdir += fGridWorkingDir;
d3b18c4c 4530 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4531 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4532// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4533 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4534 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4535 }
4536}