]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix for empty library name in SetFriendChainName
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
105631b5 27#include "TKey.h"
7c2cd90a 28#include "TBits.h"
0f389141 29#include "TError.h"
c57f56b7 30#include "TROOT.h"
31#include "TSystem.h"
32#include "TFile.h"
830acc4c 33#include "TFileCollection.h"
3bdcb562 34#include "TChain.h"
c57f56b7 35#include "TObjString.h"
36#include "TObjArray.h"
77f34eae 37#include "TMacro.h"
c57f56b7 38#include "TGrid.h"
39#include "TGridResult.h"
40#include "TGridCollection.h"
41#include "TGridJDL.h"
d2a409b2 42#include "TGridJobStatusList.h"
43#include "TGridJobStatus.h"
c57f56b7 44#include "TFileMerger.h"
45#include "AliAnalysisManager.h"
77f34eae 46#include "AliAnalysisTaskCfg.h"
bb885a9e 47#include "AliVEventHandler.h"
48#include "AliAnalysisDataContainer.h"
65eb22f8 49#include "AliMultiInputEventHandler.h"
c57f56b7 50
c82bb898 51using std::ofstream;
52using std::ifstream;
53using std::ios;
54using std::endl;
c57f56b7 55ClassImp(AliAnalysisAlien)
6c395669 56#if 0
57;
58#endif
c57f56b7 59
6c395669 60namespace {
61 Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
62 {
63 TString sl(Form("file:%s", loc));
64 TString sr(Form("alien://%s", rem));
65 Bool_t ret = TFile::Cp(sl, sr);
66 if (!ret) {
67 Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
68 }
69 return ret;
70 }
71}
72
c57f56b7 73//______________________________________________________________________________
74AliAnalysisAlien::AliAnalysisAlien()
75 :AliAnalysisGrid(),
76 fGridJDL(NULL),
0f389141 77 fMergingJDL(NULL),
c57f56b7 78 fPrice(0),
79 fTTL(0),
80 fSplitMaxInputFileNumber(0),
81 fMaxInitFailed(0),
82 fMasterResubmitThreshold(0),
bb885a9e 83 fNtestFiles(0),
319593fb 84 fNrunsPerMaster(0),
16a4353c 85 fMaxMergeFiles(0),
b3e07543 86 fMaxMergeStages(0),
d2a409b2 87 fNsubmitted(0),
a3e84053 88 fProductionMode(0),
cd11251e 89 fOutputToRunNo(0),
0f389141 90 fMergeViaJDL(0),
91 fFastReadOption(0),
e1c22e21 92 fOverwriteMode(1),
149d288c 93 fNreplicas(2),
3bdcb562 94 fNproofWorkers(0),
27734f0e 95 fNproofWorkersPerSlave(0),
3bdcb562 96 fProofReset(0),
c57f56b7 97 fRunNumbers(),
98 fExecutable(),
0a1c1f7f 99 fExecutableCommand(),
c57f56b7 100 fArguments(),
631c0b05 101 fExecutableArgs(),
c57f56b7 102 fAnalysisMacro(),
103 fAnalysisSource(),
d3b18c4c 104 fValidationScript(),
d5c6455a 105 fAdditionalRootLibs(),
c57f56b7 106 fAdditionalLibs(),
107 fSplitMode(),
108 fAPIVersion(),
109 fROOTVersion(),
110 fAliROOTVersion(),
648174cf 111 fExternalPackages(),
c57f56b7 112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
d3b18c4c 122 fTerminateFiles(),
bb885a9e 123 fMergeExcludes(),
37d21c01 124 fRegisterExcludes(),
f965131e 125 fIncludePath(),
bb885a9e 126 fCloseSE(),
0df6ccf2 127 fFriendChainName(),
c6cb3634 128 fJobTag(),
648174cf 129 fOutputSingle(),
5fce53f4 130 fRunPrefix(),
3bdcb562 131 fProofCluster(),
132 fProofDataSet(),
133 fFileForTestMode(),
3bdcb562 134 fAliRootMode(),
d88e4159 135 fProofProcessOpt(),
f47d5cb4 136 fMergeDirName(),
4e5c5506 137 fInputFiles(0),
28ce6c9c 138 fPackages(0),
77f34eae 139 fModules(0),
d88e4159 140 fProofParam(),
a41b2882 141 fDropToShell(true),
142 fGridJobIDs(""),
143 fGridStages(""),
144 fFriendLibs("")
c57f56b7 145{
146// Dummy ctor.
147 SetDefaults();
148}
149
150//______________________________________________________________________________
151AliAnalysisAlien::AliAnalysisAlien(const char *name)
152 :AliAnalysisGrid(name),
153 fGridJDL(NULL),
0f389141 154 fMergingJDL(NULL),
c57f56b7 155 fPrice(0),
156 fTTL(0),
157 fSplitMaxInputFileNumber(0),
158 fMaxInitFailed(0),
159 fMasterResubmitThreshold(0),
bb885a9e 160 fNtestFiles(0),
319593fb 161 fNrunsPerMaster(0),
16a4353c 162 fMaxMergeFiles(0),
b3e07543 163 fMaxMergeStages(0),
d2a409b2 164 fNsubmitted(0),
a3e84053 165 fProductionMode(0),
cd11251e 166 fOutputToRunNo(0),
0f389141 167 fMergeViaJDL(0),
168 fFastReadOption(0),
e1c22e21 169 fOverwriteMode(1),
149d288c 170 fNreplicas(2),
3bdcb562 171 fNproofWorkers(0),
27734f0e 172 fNproofWorkersPerSlave(0),
3bdcb562 173 fProofReset(0),
c57f56b7 174 fRunNumbers(),
175 fExecutable(),
0a1c1f7f 176 fExecutableCommand(),
c57f56b7 177 fArguments(),
631c0b05 178 fExecutableArgs(),
c57f56b7 179 fAnalysisMacro(),
180 fAnalysisSource(),
d3b18c4c 181 fValidationScript(),
d5c6455a 182 fAdditionalRootLibs(),
c57f56b7 183 fAdditionalLibs(),
184 fSplitMode(),
185 fAPIVersion(),
186 fROOTVersion(),
187 fAliROOTVersion(),
648174cf 188 fExternalPackages(),
c57f56b7 189 fUser(),
190 fGridWorkingDir(),
191 fGridDataDir(),
192 fDataPattern(),
193 fGridOutputDir(),
194 fOutputArchive(),
195 fOutputFiles(),
196 fInputFormat(),
e7c71df0 197 fDatasetName(),
c57f56b7 198 fJDLName(),
d3b18c4c 199 fTerminateFiles(),
bb885a9e 200 fMergeExcludes(),
37d21c01 201 fRegisterExcludes(),
f965131e 202 fIncludePath(),
bb885a9e 203 fCloseSE(),
0df6ccf2 204 fFriendChainName(),
c6cb3634 205 fJobTag(),
648174cf 206 fOutputSingle(),
5fce53f4 207 fRunPrefix(),
3bdcb562 208 fProofCluster(),
209 fProofDataSet(),
210 fFileForTestMode(),
3bdcb562 211 fAliRootMode(),
d88e4159 212 fProofProcessOpt(),
f47d5cb4 213 fMergeDirName(),
4e5c5506 214 fInputFiles(0),
28ce6c9c 215 fPackages(0),
77f34eae 216 fModules(0),
d88e4159 217 fProofParam(),
0ad3ea1f 218 fDropToShell(true),
219 fGridJobIDs(""),
a41b2882 220 fGridStages(""),
221 fFriendLibs("")
c57f56b7 222{
223// Default ctor.
224 SetDefaults();
225}
226
227//______________________________________________________________________________
228AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
229 :AliAnalysisGrid(other),
230 fGridJDL(NULL),
0f389141 231 fMergingJDL(NULL),
c57f56b7 232 fPrice(other.fPrice),
233 fTTL(other.fTTL),
234 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
235 fMaxInitFailed(other.fMaxInitFailed),
236 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 237 fNtestFiles(other.fNtestFiles),
319593fb 238 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 239 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 240 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 241 fNsubmitted(other.fNsubmitted),
a3e84053 242 fProductionMode(other.fProductionMode),
cd11251e 243 fOutputToRunNo(other.fOutputToRunNo),
0f389141 244 fMergeViaJDL(other.fMergeViaJDL),
245 fFastReadOption(other.fFastReadOption),
246 fOverwriteMode(other.fOverwriteMode),
149d288c 247 fNreplicas(other.fNreplicas),
3bdcb562 248 fNproofWorkers(other.fNproofWorkers),
27734f0e 249 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 250 fProofReset(other.fProofReset),
c57f56b7 251 fRunNumbers(other.fRunNumbers),
252 fExecutable(other.fExecutable),
0a1c1f7f 253 fExecutableCommand(other.fExecutableCommand),
c57f56b7 254 fArguments(other.fArguments),
631c0b05 255 fExecutableArgs(other.fExecutableArgs),
c57f56b7 256 fAnalysisMacro(other.fAnalysisMacro),
257 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 258 fValidationScript(other.fValidationScript),
d5c6455a 259 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 260 fAdditionalLibs(other.fAdditionalLibs),
261 fSplitMode(other.fSplitMode),
262 fAPIVersion(other.fAPIVersion),
263 fROOTVersion(other.fROOTVersion),
264 fAliROOTVersion(other.fAliROOTVersion),
648174cf 265 fExternalPackages(other.fExternalPackages),
c57f56b7 266 fUser(other.fUser),
267 fGridWorkingDir(other.fGridWorkingDir),
268 fGridDataDir(other.fGridDataDir),
269 fDataPattern(other.fDataPattern),
270 fGridOutputDir(other.fGridOutputDir),
271 fOutputArchive(other.fOutputArchive),
272 fOutputFiles(other.fOutputFiles),
273 fInputFormat(other.fInputFormat),
e7c71df0 274 fDatasetName(other.fDatasetName),
c57f56b7 275 fJDLName(other.fJDLName),
d3b18c4c 276 fTerminateFiles(other.fTerminateFiles),
bb885a9e 277 fMergeExcludes(other.fMergeExcludes),
37d21c01 278 fRegisterExcludes(other.fRegisterExcludes),
f965131e 279 fIncludePath(other.fIncludePath),
bb885a9e 280 fCloseSE(other.fCloseSE),
0df6ccf2 281 fFriendChainName(other.fFriendChainName),
c6cb3634 282 fJobTag(other.fJobTag),
648174cf 283 fOutputSingle(other.fOutputSingle),
5fce53f4 284 fRunPrefix(other.fRunPrefix),
3bdcb562 285 fProofCluster(other.fProofCluster),
286 fProofDataSet(other.fProofDataSet),
287 fFileForTestMode(other.fFileForTestMode),
3bdcb562 288 fAliRootMode(other.fAliRootMode),
d88e4159 289 fProofProcessOpt(other.fProofProcessOpt),
f47d5cb4 290 fMergeDirName(other.fMergeDirName),
4e5c5506 291 fInputFiles(0),
28ce6c9c 292 fPackages(0),
77f34eae 293 fModules(0),
d88e4159 294 fProofParam(),
0ad3ea1f 295 fDropToShell(other.fDropToShell),
296 fGridJobIDs(other.fGridJobIDs),
a41b2882 297 fGridStages(other.fGridStages),
298 fFriendLibs(other.fFriendLibs)
c57f56b7 299{
300// Copy ctor.
301 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 302 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 303 fRunRange[0] = other.fRunRange[0];
304 fRunRange[1] = other.fRunRange[1];
c57f56b7 305 if (other.fInputFiles) {
306 fInputFiles = new TObjArray();
307 TIter next(other.fInputFiles);
308 TObject *obj;
309 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
310 fInputFiles->SetOwner();
311 }
4e5c5506 312 if (other.fPackages) {
313 fPackages = new TObjArray();
314 TIter next(other.fPackages);
315 TObject *obj;
316 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
317 fPackages->SetOwner();
318 }
77f34eae 319 if (other.fModules) {
320 fModules = new TObjArray();
321 fModules->SetOwner();
322 TIter next(other.fModules);
323 AliAnalysisTaskCfg *mod, *crt;
324 while ((crt=(AliAnalysisTaskCfg*)next())) {
325 mod = new AliAnalysisTaskCfg(*crt);
326 fModules->Add(mod);
327 }
328 }
c57f56b7 329}
330
331//______________________________________________________________________________
332AliAnalysisAlien::~AliAnalysisAlien()
333{
334// Destructor.
77f34eae 335 delete fGridJDL;
336 delete fMergingJDL;
337 delete fInputFiles;
338 delete fPackages;
339 delete fModules;
28ce6c9c 340 fProofParam.DeleteAll();
c57f56b7 341}
342
343//______________________________________________________________________________
344AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
345{
346// Assignment.
347 if (this != &other) {
348 AliAnalysisGrid::operator=(other);
349 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 350 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 351 fPrice = other.fPrice;
352 fTTL = other.fTTL;
353 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
354 fMaxInitFailed = other.fMaxInitFailed;
355 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 356 fNtestFiles = other.fNtestFiles;
a3e84053 357 fNrunsPerMaster = other.fNrunsPerMaster;
358 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 359 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 360 fNsubmitted = other.fNsubmitted;
361 fProductionMode = other.fProductionMode;
cd11251e 362 fOutputToRunNo = other.fOutputToRunNo;
0f389141 363 fMergeViaJDL = other.fMergeViaJDL;
364 fFastReadOption = other.fFastReadOption;
365 fOverwriteMode = other.fOverwriteMode;
149d288c 366 fNreplicas = other.fNreplicas;
3bdcb562 367 fNproofWorkers = other.fNproofWorkers;
27734f0e 368 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 369 fProofReset = other.fProofReset;
c57f56b7 370 fRunNumbers = other.fRunNumbers;
371 fExecutable = other.fExecutable;
0a1c1f7f 372 fExecutableCommand = other.fExecutableCommand;
c57f56b7 373 fArguments = other.fArguments;
631c0b05 374 fExecutableArgs = other.fExecutableArgs;
c57f56b7 375 fAnalysisMacro = other.fAnalysisMacro;
376 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 377 fValidationScript = other.fValidationScript;
d5c6455a 378 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 379 fAdditionalLibs = other.fAdditionalLibs;
380 fSplitMode = other.fSplitMode;
381 fAPIVersion = other.fAPIVersion;
382 fROOTVersion = other.fROOTVersion;
383 fAliROOTVersion = other.fAliROOTVersion;
648174cf 384 fExternalPackages = other.fExternalPackages;
c57f56b7 385 fUser = other.fUser;
386 fGridWorkingDir = other.fGridWorkingDir;
387 fGridDataDir = other.fGridDataDir;
388 fDataPattern = other.fDataPattern;
389 fGridOutputDir = other.fGridOutputDir;
390 fOutputArchive = other.fOutputArchive;
391 fOutputFiles = other.fOutputFiles;
392 fInputFormat = other.fInputFormat;
e7c71df0 393 fDatasetName = other.fDatasetName;
c57f56b7 394 fJDLName = other.fJDLName;
d3b18c4c 395 fTerminateFiles = other.fTerminateFiles;
bb885a9e 396 fMergeExcludes = other.fMergeExcludes;
37d21c01 397 fRegisterExcludes = other.fRegisterExcludes;
f965131e 398 fIncludePath = other.fIncludePath;
bb885a9e 399 fCloseSE = other.fCloseSE;
0df6ccf2 400 fFriendChainName = other.fFriendChainName;
c6cb3634 401 fJobTag = other.fJobTag;
648174cf 402 fOutputSingle = other.fOutputSingle;
5fce53f4 403 fRunPrefix = other.fRunPrefix;
3bdcb562 404 fProofCluster = other.fProofCluster;
405 fProofDataSet = other.fProofDataSet;
406 fFileForTestMode = other.fFileForTestMode;
3bdcb562 407 fAliRootMode = other.fAliRootMode;
d88e4159 408 fProofProcessOpt = other.fProofProcessOpt;
f47d5cb4 409 fMergeDirName = other.fMergeDirName;
d88e4159 410 fDropToShell = other.fDropToShell;
0ad3ea1f 411 fGridJobIDs = other.fGridJobIDs;
412 fGridStages = other.fGridStages;
a41b2882 413 fFriendLibs = other.fFriendLibs;
c57f56b7 414 if (other.fInputFiles) {
415 fInputFiles = new TObjArray();
416 TIter next(other.fInputFiles);
417 TObject *obj;
418 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
419 fInputFiles->SetOwner();
420 }
4e5c5506 421 if (other.fPackages) {
422 fPackages = new TObjArray();
423 TIter next(other.fPackages);
424 TObject *obj;
425 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
426 fPackages->SetOwner();
427 }
77f34eae 428 if (other.fModules) {
429 fModules = new TObjArray();
430 fModules->SetOwner();
431 TIter next(other.fModules);
432 AliAnalysisTaskCfg *mod, *crt;
433 while ((crt=(AliAnalysisTaskCfg*)next())) {
434 mod = new AliAnalysisTaskCfg(*crt);
435 fModules->Add(mod);
436 }
437 }
c57f56b7 438 }
439 return *this;
440}
441
37d21c01 442//______________________________________________________________________________
443void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
444{
445// Add a single additional library to be loaded. Extension must be present.
446 TString lib(name);
447 if (!lib.Contains(".")) {
448 Error("AddAdditionalLibrary", "Extension not defined for %s", name);
449 return;
450 }
451 if (fAdditionalLibs.Contains(name)) {
452 Warning("AddAdditionalLibrary", "Library %s already added.", name);
453 return;
454 }
455 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
456 fAdditionalLibs += lib;
457}
458
77f34eae 459//______________________________________________________________________________
460void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
461{
462// Adding a module. Checks if already existing. Becomes owned by this.
463 if (!module) return;
464 if (GetModule(module->GetName())) {
465 Error("AddModule", "A module having the same name %s already added", module->GetName());
466 return;
467 }
468 if (!fModules) {
469 fModules = new TObjArray();
470 fModules->SetOwner();
471 }
472 fModules->Add(module);
473}
474
475//______________________________________________________________________________
476void AliAnalysisAlien::AddModules(TObjArray *list)
477{
478// Adding a list of modules. Checks if already existing. Becomes owned by this.
479 TIter next(list);
480 AliAnalysisTaskCfg *module;
481 while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
482}
483
484//______________________________________________________________________________
485Bool_t AliAnalysisAlien::CheckDependencies()
486{
487// Check if all dependencies are satisfied. Reorder modules if needed.
488 Int_t nmodules = GetNmodules();
489 if (!nmodules) {
490 Warning("CheckDependencies", "No modules added yet to check their dependencies");
491 return kTRUE;
492 }
493 AliAnalysisTaskCfg *mod = 0;
494 AliAnalysisTaskCfg *dep = 0;
495 TString depname;
496 Int_t i, j, k;
497 for (i=0; i<nmodules; i++) {
498 mod = (AliAnalysisTaskCfg*) fModules->At(i);
499 Int_t ndeps = mod->GetNdeps();
500 Int_t istart = i;
501 for (j=0; j<ndeps; j++) {
502 depname = mod->GetDependency(j);
503 dep = GetModule(depname);
504 if (!dep) {
505 Error("CheckDependencies","Dependency %s not added for module %s",
506 depname.Data(), mod->GetName());
507 return kFALSE;
508 }
509 if (dep->NeedsDependency(mod->GetName())) {
510 Error("CheckDependencies","Modules %s and %s circularly depend on each other",
511 mod->GetName(), dep->GetName());
512 return kFALSE;
513 }
514 Int_t idep = fModules->IndexOf(dep);
515 // The dependency task must come first
516 if (idep>i) {
517 // Remove at idep and move all objects below up one slot
518 // down to index i included.
519 fModules->RemoveAt(idep);
3e40fd4c 520 for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
77f34eae 521 fModules->AddAt(dep, i++);
522 }
523 //Redo from istart if dependencies were inserted
524 if (i>istart) i=istart-1;
525 }
526 }
527 return kTRUE;
528}
529
3e40fd4c 530//______________________________________________________________________________
531AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
532{
533// Create the analysis manager and optionally execute the macro in filename.
534 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
535 if (mgr) return mgr;
536 mgr = new AliAnalysisManager(name);
537 mgr->SetGridHandler((AliAnalysisGrid*)this);
538 if (strlen(filename)) {
539 TString line = gSystem->ExpandPathName(filename);
540 line.Prepend(".x ");
541 gROOT->ProcessLine(line.Data());
542 }
543 return mgr;
544}
545
77f34eae 546//______________________________________________________________________________
547Int_t AliAnalysisAlien::GetNmodules() const
548{
549// Get number of modules.
550 if (!fModules) return 0;
551 return fModules->GetEntries();
552}
553
554//______________________________________________________________________________
555AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
556{
557// Get a module by name.
558 if (!fModules) return 0;
559 return (AliAnalysisTaskCfg*)fModules->FindObject(name);
560}
561
562//______________________________________________________________________________
563Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
564{
565// Load a given module.
566 if (mod->IsLoaded()) return kTRUE;
192ddca2 567 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
568 if (!mgr) {
569 Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
570 return kFALSE;
571 }
77f34eae 572 Int_t ndeps = mod->GetNdeps();
573 TString depname;
574 for (Int_t j=0; j<ndeps; j++) {
575 depname = mod->GetDependency(j);
576 AliAnalysisTaskCfg *dep = GetModule(depname);
577 if (!dep) {
578 Error("LoadModule","Dependency %s not existing for module %s",
579 depname.Data(), mod->GetName());
580 return kFALSE;
581 }
582 if (!LoadModule(dep)) {
583 Error("LoadModule","Dependency %s for module %s could not be loaded",
584 depname.Data(), mod->GetName());
585 return kFALSE;
586 }
587 }
588 // Load libraries for the module
589 if (!mod->CheckLoadLibraries()) {
590 Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
591 return kFALSE;
592 }
192ddca2 593 // Check if a custom file name was requested
594 if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
595
596 // Check if a custom terminate file name was requested
597 if (strlen(mod->GetTerminateFileName())) {
598 if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
599 fTerminateFiles += mod->GetTerminateFileName();
600 }
601
77f34eae 602 // Execute the macro
603 if (mod->ExecuteMacro()<0) {
604 Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
605 mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
606 return kFALSE;
607 }
608 // Configure dependencies
609 if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
610 Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
611 mod->GetConfigMacro()->GetTitle(), mod->GetName());
612 return kFALSE;
613 }
5e8c7c22 614 // Adjust extra libraries
615 Int_t nlibs = mod->GetNlibs();
616 TString lib;
617 for (Int_t i=0; i<nlibs; i++) {
618 lib = mod->GetLibrary(i);
5e8c7c22 619 lib = Form("lib%s.so", lib.Data());
c9d6bd12 620 if (fAdditionalLibs.Contains(lib)) continue;
5e8c7c22 621 if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
622 fAdditionalLibs += lib;
623 }
77f34eae 624 return kTRUE;
625}
626
139fe6ba 627//______________________________________________________________________________
628Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
629{
630// Generate the full train.
631 fAdditionalLibs = "";
632 if (!LoadModules()) return kFALSE;
633 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
634 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 635 mgr->RunLocalInit();
139fe6ba 636 mgr->PrintStatus();
637 Int_t productionMode = fProductionMode;
638 SetProductionMode();
639 TString macro = fAnalysisMacro;
640 TString executable = fExecutable;
641 TString validation = fValidationScript;
642 TString execCommand = fExecutableCommand;
643 SetAnalysisMacro(Form("%s.C", name));
644 SetExecutable(Form("%s.sh", name));
a8e31fb0 645// SetExecutableCommand("aliroot -b -q ");
139fe6ba 646 SetValidationScript(Form("%s_validation.sh", name));
647 StartAnalysis();
648 SetProductionMode(productionMode);
649 fAnalysisMacro = macro;
650 fExecutable = executable;
651 fExecutableCommand = execCommand;
652 fValidationScript = validation;
653 return kTRUE;
654}
655
77f34eae 656//______________________________________________________________________________
21ca8e59 657Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
77f34eae 658{
659// Generate test macros for a single module or for the full train.
5e8c7c22 660 fAdditionalLibs = "";
77f34eae 661 if (strlen(modname)) {
21ca8e59 662 if (!CheckDependencies()) return kFALSE;
663 AliAnalysisTaskCfg *mod = GetModule(modname);
664 if (!mod) {
665 Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
666 return kFALSE;
667 }
668 if (!LoadModule(mod)) return kFALSE;
a41b2882 669 if (!LoadFriendLibs()) return kFALSE;
21ca8e59 670 } else if (!LoadModules()) return kFALSE;
671 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
672 if (!mgr->InitAnalysis()) return kFALSE;
d512d8e2 673 mgr->RunLocalInit();
21ca8e59 674 mgr->PrintStatus();
675 SetLocalTest(kTRUE);
676 Int_t productionMode = fProductionMode;
677 SetProductionMode();
678 TString macro = fAnalysisMacro;
679 TString executable = fExecutable;
680 TString validation = fValidationScript;
681 TString execCommand = fExecutableCommand;
682 SetAnalysisMacro(Form("%s.C", name));
683 SetExecutable(Form("%s.sh", name));
d66aa2a9 684 fOutputFiles = GetListOfFiles("outaod");
685 // Add extra files registered to the analysis manager
686 TString extra = GetListOfFiles("ext");
687 if (!extra.IsNull()) {
688 extra.ReplaceAll(".root", "*.root");
689 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
690 fOutputFiles += extra;
691 }
a8e31fb0 692// SetExecutableCommand("aliroot -b -q ");
21ca8e59 693 SetValidationScript(Form("%s_validation.sh", name));
694 WriteAnalysisFile();
695 WriteAnalysisMacro();
696 WriteExecutable();
697 WriteValidationScript();
ffbe06b9 698 WriteMergingMacro();
699 WriteMergeExecutable();
700 WriteValidationScript(kTRUE);
21ca8e59 701 SetLocalTest(kFALSE);
702 SetProductionMode(productionMode);
703 fAnalysisMacro = macro;
704 fExecutable = executable;
705 fExecutableCommand = execCommand;
706 fValidationScript = validation;
77f34eae 707 return kTRUE;
708}
709
710//______________________________________________________________________________
711Bool_t AliAnalysisAlien::LoadModules()
712{
713// Load all modules by executing the AddTask macros. Checks first the dependencies.
5e8c7c22 714 fAdditionalLibs = "";
21ca8e59 715 Int_t nmodules = GetNmodules();
716 if (!nmodules) {
717 Warning("LoadModules", "No module to be loaded");
718 return kTRUE;
719 }
3e40fd4c 720 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
721 if (!mgr) {
722 Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
723 return kFALSE;
724 }
77f34eae 725 if (!CheckDependencies()) return kFALSE;
21ca8e59 726 nmodules = GetNmodules();
77f34eae 727 AliAnalysisTaskCfg *mod;
21ca8e59 728 for (Int_t imod=0; imod<nmodules; imod++) {
729 mod = (AliAnalysisTaskCfg*)fModules->At(imod);
77f34eae 730 if (!LoadModule(mod)) return kFALSE;
731 }
a41b2882 732 // Load additional friend libraries
733 return LoadFriendLibs();
77f34eae 734}
735
a41b2882 736//______________________________________________________________________________
737Bool_t AliAnalysisAlien::LoadFriendLibs() const
738{
739// Load libraries required for reading friends.
740 if (fFriendLibs.Length()) {
741 TObjArray *list = 0;
742 TString lib;
743 if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
744 else list = fFriendLibs.Tokenize(" ");
745 for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
746 lib = list->At(ilib)->GetName();
747 lib.ReplaceAll(".so","");
748 lib.ReplaceAll(" ","");
749 if (lib.BeginsWith("lib")) lib.Remove(0, 3);
750 lib.Prepend("lib");
751 Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
752 if (!loaded) loaded = gSystem->Load(lib);
753 if (loaded < 0) {
754 Error("LoadModules", "Cannot load library for friends %s", lib.Data());
755 return kFALSE;
756 }
757 }
758 delete list;
759 }
760 return kTRUE;
761}
762
74013535 763//______________________________________________________________________________
764void AliAnalysisAlien::SetRunPrefix(const char *prefix)
765{
766// Set the run number format. Can be a prefix or a format like "%09d"
767 fRunPrefix = prefix;
768 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
769}
770
f965131e 771//______________________________________________________________________________
772void AliAnalysisAlien::AddIncludePath(const char *path)
773{
774// Add include path in the remote analysis macro.
775 TString p(path);
776 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
777 else fIncludePath += Form("-I%s ", path);
778}
779
c57f56b7 780//______________________________________________________________________________
781void AliAnalysisAlien::AddRunNumber(Int_t run)
782{
783// Add a run number to the list of runs to be processed.
784 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 785 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 786}
787
98690895 788//______________________________________________________________________________
789void AliAnalysisAlien::AddRunList(const char* runList)
790{
791// Add several runs into the list of runs; they are expected to be separated by a blank character.
792 TString sList = runList;
793 TObjArray *list = sList.Tokenize(" ");
794 Int_t n = list->GetEntries();
795 for (Int_t i = 0; i < n; i++) {
796 TObjString *os = (TObjString*)list->At(i);
797 AddRunNumber(os->GetString().Atoi());
798 }
799 delete list;
800}
801
ee75cfc3 802//______________________________________________________________________________
803void AliAnalysisAlien::AddRunNumber(const char* run)
804{
805// Add a run number to the list of runs to be processed.
7aa5d539 806 TString runs = run;
807 TObjString *os;
808 TObjArray *arr = runs.Tokenize(" ");
809 TIter next(arr);
810 TString prefix;
811 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
812 while ((os=(TObjString*)next())){
813 if (fRunNumbers.Length()) fRunNumbers += " ";
814 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
815 }
816 delete arr;
ee75cfc3 817}
818
c57f56b7 819//______________________________________________________________________________
820void AliAnalysisAlien::AddDataFile(const char *lfn)
821{
822// Adds a data file to the input to be analysed. The file should be a valid LFN
823// or point to an existing file in the alien workdir.
824 if (!fInputFiles) fInputFiles = new TObjArray();
825 fInputFiles->Add(new TObjString(lfn));
826}
648174cf 827
828//______________________________________________________________________________
829void AliAnalysisAlien::AddExternalPackage(const char *package)
830{
831// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
832 if (fExternalPackages) fExternalPackages += " ";
833 fExternalPackages += package;
834}
835
c57f56b7 836//______________________________________________________________________________
837Bool_t AliAnalysisAlien::Connect()
838{
839// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
840 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 841 if (fProductionMode) return kTRUE;
c57f56b7 842 if (!gGrid) {
843 Info("Connect", "Trying to connect to AliEn ...");
844 TGrid::Connect("alien://");
845 }
846 if (!gGrid || !gGrid->IsConnected()) {
847 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
848 return kFALSE;
849 }
850 fUser = gGrid->GetUser();
851 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
852 return kTRUE;
853}
854
855//______________________________________________________________________________
856void AliAnalysisAlien::CdWork()
857{
858// Check validity of alien workspace. Create directory if possible.
859 if (!Connect()) {
860 Error("CdWork", "Alien connection required");
861 return;
862 }
863 TString homedir = gGrid->GetHomeDirectory();
864 TString workdir = homedir + fGridWorkingDir;
923e2ca5 865 if (DirectoryExists(workdir)) {
866 gGrid->Cd(workdir);
867 return;
868 }
869 // Work directory not existing - create it
870 gGrid->Cd(homedir);
b93f8109 871 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 872 gGrid->Cd(fGridWorkingDir);
d3b18c4c 873 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 874 } else {
d3b18c4c 875 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 876 workdir.Data(), homedir.Data());
877 fGridWorkingDir = "";
878 }
c57f56b7 879}
880
348be253 881//______________________________________________________________________________
882Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
883{
884// Check if file copying is possible.
d3b18c4c 885 if (fProductionMode) return kTRUE;
b5ae4fe0 886 TString salienpath(alienpath);
887 if (salienpath.Contains(" ")) {
888 Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
889 return kFALSE;
890 }
348be253 891 if (!Connect()) {
892 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
893 return kFALSE;
894 }
d3339be3 895 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
896 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 897 // Check if alien_CLOSE_SE is defined
898 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
899 if (!closeSE.IsNull()) {
900 Info("CheckFileCopy", "Your current close storage is pointing to: \
901 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
902 } else {
903 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
904 }
905 // Check if grid directory exists.
906 if (!DirectoryExists(alienpath)) {
907 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
908 return kFALSE;
909 }
b5ae4fe0 910 TString stest = "plugin_test_copy";
911 TFile f(stest, "RECREATE");
348be253 912 // User may not have write permissions to current directory
913 if (f.IsZombie()) {
914 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
915 gSystem->WorkingDirectory());
916 return kFALSE;
917 }
918 f.Close();
b5ae4fe0 919 if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
920 if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
d3339be3 921 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 922 \n# 1. Make sure you have write permissions there. If this is the case: \
923 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
924 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
925 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
926 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
b5ae4fe0 927 gSystem->Unlink(stest.Data());
348be253 928 return kFALSE;
929 }
b5ae4fe0 930 gSystem->Unlink(stest.Data());
931 gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
d3339be3 932 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 933 return kTRUE;
934}
935
c57f56b7 936//______________________________________________________________________________
937Bool_t AliAnalysisAlien::CheckInputData()
938{
939// Check validity of input data. If necessary, create xml files.
d3b18c4c 940 if (fProductionMode) return kTRUE;
a8739e8a 941 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 942 if (!fGridDataDir.Length()) {
943 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
944 return kFALSE;
945 }
9f020952 946 if (fMergeViaJDL) {
947 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
948 return kFALSE;
949 }
d2a409b2 950 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 951 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
952 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 953 return kTRUE;
c57f56b7 954 }
955 // Process declared files
a2f5fc01 956 Bool_t isCollection = kFALSE;
957 Bool_t isXml = kFALSE;
958 Bool_t useTags = kFALSE;
c57f56b7 959 Bool_t checked = kFALSE;
d3b18c4c 960 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 961 TString file;
962 TString workdir = gGrid->GetHomeDirectory();
963 workdir += fGridWorkingDir;
964 if (fInputFiles) {
965 TObjString *objstr;
966 TIter next(fInputFiles);
967 while ((objstr=(TObjString*)next())) {
968 file = workdir;
969 file += "/";
970 file += objstr->GetString();
971 // Store full lfn path
972 if (FileExists(file)) objstr->SetString(file);
973 else {
974 file = objstr->GetName();
975 if (!FileExists(objstr->GetName())) {
976 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
977 objstr->GetName(), workdir.Data());
978 return kFALSE;
979 }
980 }
981 Bool_t iscoll, isxml, usetags;
982 CheckDataType(file, iscoll, isxml, usetags);
983 if (!checked) {
984 checked = kTRUE;
a2f5fc01 985 isCollection = iscoll;
986 isXml = isxml;
987 useTags = usetags;
988 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 989 } else {
a2f5fc01 990 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 991 Error("CheckInputData", "Some conflict was found in the types of inputs");
992 return kFALSE;
993 }
994 }
995 }
996 }
997 // Process requested run numbers
a8739e8a 998 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 999 // Check validity of alien data directory
1000 if (!fGridDataDir.Length()) {
1001 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1002 return kFALSE;
1003 }
923e2ca5 1004 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 1005 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1006 return kFALSE;
1007 }
a2f5fc01 1008 if (isCollection) {
c57f56b7 1009 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1010 return kFALSE;
1011 }
1012
a2f5fc01 1013 if (checked && !isXml) {
c57f56b7 1014 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1015 return kFALSE;
1016 }
1017 // Check validity of run number(s)
1018 TObjArray *arr;
1019 TObjString *os;
74013535 1020 TString format;
319593fb 1021 Int_t nruns = 0;
904f9f5f 1022 TString schunk, schunk2;
c57f56b7 1023 TString path;
1024 if (!checked) {
1025 checked = kTRUE;
a2f5fc01 1026 useTags = fDataPattern.Contains("tag");
1027 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 1028 }
a2f5fc01 1029 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 1030 Error("CheckInputData", "Cannot mix input files using/not using tags");
1031 return kFALSE;
1032 }
1033 if (fRunNumbers.Length()) {
a8739e8a 1034 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 1035 arr = fRunNumbers.Tokenize(" ");
1036 TIter next(arr);
1037 while ((os=(TObjString*)next())) {
1038 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1039 if (!DirectoryExists(path)) {
1040 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 1041 continue;
c57f56b7 1042 }
1043 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1044 TString msg = "\n##### file: ";
1045 msg += path;
1046 msg += " type: xml_collection;";
a2f5fc01 1047 if (useTags) msg += " using_tags: Yes";
c57f56b7 1048 else msg += " using_tags: No";
23329835 1049 Info("CheckDataType", "%s", msg.Data());
319593fb 1050 if (fNrunsPerMaster<2) {
d2a409b2 1051 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 1052 } else {
1053 nruns++;
1054 if (((nruns-1)%fNrunsPerMaster) == 0) {
1055 schunk = os->GetString();
1056 }
1057 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1058 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 1059 AddDataFile(schunk);
319593fb 1060 }
c57f56b7 1061 }
1062 delete arr;
a8739e8a 1063 } else {
1064 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1065 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1066 format = Form("%%s/%s ", fRunPrefix.Data());
1067 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1068 if (!DirectoryExists(path)) {
a8739e8a 1069 continue;
1070 }
74013535 1071 format = Form("%%s/%s.xml", fRunPrefix.Data());
1072 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 1073 TString msg = "\n##### file: ";
1074 msg += path;
1075 msg += " type: xml_collection;";
a2f5fc01 1076 if (useTags) msg += " using_tags: Yes";
a8739e8a 1077 else msg += " using_tags: No";
23329835 1078 Info("CheckDataType", "%s", msg.Data());
319593fb 1079 if (fNrunsPerMaster<2) {
74013535 1080 format = Form("%s.xml", fRunPrefix.Data());
1081 AddDataFile(Form(format.Data(),irun));
319593fb 1082 } else {
1083 nruns++;
1084 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1085 schunk = Form(fRunPrefix.Data(),irun);
319593fb 1086 }
74013535 1087 format = Form("_%s.xml", fRunPrefix.Data());
1088 schunk2 = Form(format.Data(), irun);
319593fb 1089 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 1090 schunk += schunk2;
d2a409b2 1091 AddDataFile(schunk);
319593fb 1092 }
a8739e8a 1093 }
904f9f5f 1094 if (!fInputFiles) {
1095 schunk += schunk2;
1096 AddDataFile(schunk);
1097 }
c57f56b7 1098 }
1099 return kTRUE;
1100}
1101
21ca8e59 1102//______________________________________________________________________________
4a8b667c 1103Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
21ca8e59 1104{
1105// Copy data from the given grid directory according a pattern and make a local
1106// dataset.
2ba4c287 1107// archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
21ca8e59 1108 if (!Connect()) {
1109 Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
4a8b667c 1110 return 0;
21ca8e59 1111 }
1112 if (!DirectoryExists(griddir)) {
1113 Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
4a8b667c 1114 return 0;
21ca8e59 1115 }
1116 TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1117 printf("Running command: %s\n", command.Data());
1118 TGridResult *res = gGrid->Command(command);
1119 Int_t nfound = res->GetEntries();
1120 if (!nfound) {
1121 Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
4a8b667c 1122 return 0;
21ca8e59 1123 }
1124 printf("... found %d files. Copying locally ...\n", nfound);
2ba4c287 1125
1126 // archives
1127 TObjArray* additionalArchives = 0;
1128 if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1129 additionalArchives = TString(archivefile).Tokenize(";");
1130 archivefile = additionalArchives->At(0)->GetName();
1131 additionalArchives->RemoveAt(0);
1132 additionalArchives->Compress();
1133 }
1134
21ca8e59 1135 // Copy files locally
1136 ofstream out;
1137 out.open(output, ios::out);
1138 TMap *map;
1139 TString turl, dirname, filename, temp;
1140 TString cdir = gSystem->WorkingDirectory();
139fe6ba 1141 gSystem->MakeDirectory(outputdir);
1142 gSystem->ChangeDirectory(outputdir);
4a8b667c 1143 Int_t ncopied = 0;
21ca8e59 1144 for (Int_t i=0; i<nfound; i++) {
1145 map = (TMap*)res->At(i);
1146 turl = map->GetValue("turl")->GetName();
1147 filename = gSystem->BaseName(turl.Data());
1148 dirname = gSystem->DirName(turl.Data());
1149 dirname = gSystem->BaseName(dirname.Data());
1150 gSystem->MakeDirectory(dirname);
43b807f6 1151
1152 TString source(turl);
1153 TString targetFileName(filename);
1154
1155 if (strlen(archivefile) > 0) {
4a8b667c 1156// TODO here the archive in which the file resides should be determined
1157// however whereis returns only a guid, and guid2lfn does not work
1158// Therefore we use the one provided as argument for now
1159 source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1160 targetFileName = archivefile;
43b807f6 1161 }
1162 if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
4a8b667c 1163 Bool_t success = kTRUE;
1164 if (additionalArchives) {
1165 for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1166 TString target;
1167 target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1168 gSystem->MakeDirectory(gSystem->DirName(target));
1169 success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1170 }
1171 }
2ba4c287 1172
4a8b667c 1173 if (success) {
1174 if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1175 out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1176 ncopied++;
1177 }
21ca8e59 1178 }
1179 }
1180 gSystem->ChangeDirectory(cdir);
1181 delete res;
4a8b667c 1182 delete additionalArchives;
1183 return ncopied;
21ca8e59 1184}
1185
c57f56b7 1186//______________________________________________________________________________
1187Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1188{
1189// Create dataset for the grid data directory + run number.
3af861ff 1190 const Int_t gMaxEntries = 15000;
d3b18c4c 1191 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 1192 if (!Connect()) {
1193 Error("CreateDataset", "Cannot create dataset with no grid connection");
1194 return kFALSE;
1195 }
1196
1197 // Cd workspace
d3b18c4c 1198 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1199 TString workdir = gGrid->GetHomeDirectory();
1200 workdir += fGridWorkingDir;
1201
1202 // Compose the 'find' command arguments
74013535 1203 TString format;
c57f56b7 1204 TString command;
1205 TString options = "-x collection ";
bb885a9e 1206 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
3af861ff 1207 else options += Form("-l %d ", gMaxEntries); // Protection for the find command
c57f56b7 1208 TString conditions = "";
3af861ff 1209 Int_t nstart = 0;
1210 Int_t ncount = 0;
1211 Int_t stage = 0;
c57f56b7 1212 TString file;
1213 TString path;
319593fb 1214 Int_t nruns = 0;
904f9f5f 1215 TString schunk, schunk2;
ab254fd1 1216 TGridCollection *cbase=0, *cadd=0;
d2a409b2 1217 if (!fRunNumbers.Length() && !fRunRange[0]) {
1218 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1219 // Make a single data collection from data directory.
1220 path = fGridDataDir;
923e2ca5 1221 if (!DirectoryExists(path)) {
d2a409b2 1222 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1223 return kFALSE;
1224 }
923e2ca5 1225// CdWork();
d2a409b2 1226 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1227 else file = Form("%s.xml", gSystem->BaseName(path));
3af861ff 1228 while (1) {
1229 ncount = 0;
1230 stage++;
1231 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1232 command = "find ";
1233 command += Form("%s -o %d ",options.Data(), nstart);
1234 command += path;
1235 command += " ";
1236 command += pattern;
1237 command += conditions;
1238 printf("command: %s\n", command.Data());
1239 TGridResult *res = gGrid->Command(command);
1240 if (res) delete res;
1241 // Write standard output to file
1242 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1243 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1244 Bool_t nullFile = kFALSE;
1245 if (!hasGrep) {
1246 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1247 } else {
1248 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1249 if (nullFile) {
1250 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1251 gSystem->Exec("rm -f __tmp*");
1252 return kFALSE;
1253 }
1254 TString line;
1255 ifstream in;
1256 in.open("__tmp__");
1257 in >> line;
1258 in.close();
1259 gSystem->Exec("rm -f __tmp__");
1260 ncount = line.Atoi();
1261 }
1262 }
1263 if (ncount == gMaxEntries) {
1264 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1265 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1266 if (!cbase) cbase = cadd;
1267 else {
1268 cbase->Add(cadd);
1269 delete cadd;
defd7a3a 1270 }
3af861ff 1271 nstart += ncount;
1272 } else {
1273 if (cbase) {
1274 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1275 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1276 cbase->Add(cadd);
1277 delete cadd;
1278 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1279 delete cbase; cbase = 0;
1280 } else {
1281 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1282 }
1283 gSystem->Exec("rm -f __tmp*");
1284 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1285 break;
1286 }
0f389141 1287 }
1288 Bool_t fileExists = FileExists(file);
1289 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 1290 // Copy xml file to alien space
0f389141 1291 if (fileExists) gGrid->Rm(file);
d2a409b2 1292 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1293 if (!FileExists(file)) {
1294 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1295 return kFALSE;
1296 }
1297 // Update list of files to be processed.
1298 }
1299 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1300 return kTRUE;
1301 }
c57f56b7 1302 // Several runs
a2f5fc01 1303 Bool_t nullResult = kTRUE;
a8739e8a 1304 if (fRunNumbers.Length()) {
1305 TObjArray *arr = fRunNumbers.Tokenize(" ");
1306 TObjString *os;
1307 TIter next(arr);
1308 while ((os=(TObjString*)next())) {
3af861ff 1309 nstart = 0;
1310 stage = 0;
dc030a24 1311 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 1312 if (!DirectoryExists(path)) continue;
1313// CdWork();
a8739e8a 1314 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1315 else file = Form("%s.xml", os->GetString().Data());
319593fb 1316 // If local collection file does not exist, create it via 'find' command.
3af861ff 1317 while (1) {
1318 ncount = 0;
1319 stage++;
1320 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1321 command = "find ";
1322 command += Form("%s -o %d ",options.Data(), nstart);
1323 command += path;
1324 command += pattern;
1325 command += conditions;
1326 TGridResult *res = gGrid->Command(command);
1327 if (res) delete res;
1328 // Write standard output to file
1329 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1330 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1331 Bool_t nullFile = kFALSE;
1332 if (!hasGrep) {
1333 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1334 } else {
1335 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1336 if (nullFile) {
1337 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1338 gSystem->Exec("rm -f __tmp*");
1339 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1604c9ed 1340 break;
3af861ff 1341 }
1342 TString line;
1343 ifstream in;
1344 in.open("__tmp__");
1345 in >> line;
1346 in.close();
1347 gSystem->Exec("rm -f __tmp__");
1348 ncount = line.Atoi();
1349 }
1350 nullResult = kFALSE;
1351 }
1352 if (ncount == gMaxEntries) {
1353 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1354 if (fNrunsPerMaster > 1) {
1355 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1356 file.Data(),gMaxEntries);
1357 return kFALSE;
1358 }
1359 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1360 if (!cbase) cbase = cadd;
1361 else {
1362 cbase->Add(cadd);
1363 delete cadd;
defd7a3a 1364 }
3af861ff 1365 nstart += ncount;
1366 } else {
c89a84bc 1367 if (cbase && fNrunsPerMaster<2) {
3af861ff 1368 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1369 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1370 cbase->Add(cadd);
1371 delete cadd;
1372 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1373 delete cbase; cbase = 0;
1374 } else {
1375 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1376 }
1377 gSystem->Exec("rm -f __tmp*");
1378 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1379 break;
d3339be3 1380 }
3af861ff 1381 }
a8739e8a 1382 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1383 // Check if there is one run per master job.
1384 if (fNrunsPerMaster<2) {
1385 if (FileExists(file)) {
0f389141 1386 if (fOverwriteMode) gGrid->Rm(file);
1387 else {
1388 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1389 continue;
1390 }
319593fb 1391 }
1392 // Copy xml file to alien space
1393 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1394 if (!FileExists(file)) {
1395 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1396 delete arr;
1397 return kFALSE;
1398 }
1399 } else {
1400 nruns++;
1401 if (((nruns-1)%fNrunsPerMaster) == 0) {
1402 schunk = os->GetString();
1403 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1404 } else {
1405 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 1406 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 1407 cbase->Add(cadd);
1408 delete cadd;
1409 }
1410 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1411 continue;
1412 }
1413 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 1414 if (FileExists(schunk)) {
1415 if (fOverwriteMode) gGrid->Rm(file);
1416 else {
1417 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1418 continue;
1419 }
319593fb 1420 }
84fcd93f 1421 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 1422 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 1423 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 1424 if (!FileExists(schunk)) {
1425 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1426 delete arr;
1427 return kFALSE;
1428 }
d3339be3 1429 }
a8739e8a 1430 }
1431 delete arr;
a2f5fc01 1432 if (nullResult) {
d3339be3 1433 Error("CreateDataset", "No valid dataset corresponding to the query!");
1434 return kFALSE;
1435 }
a8739e8a 1436 } else {
1437 // Process a full run range.
1438 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 1439 format = Form("%%s/%s ", fRunPrefix.Data());
3af861ff 1440 nstart = 0;
1441 stage = 0;
74013535 1442 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 1443 if (!DirectoryExists(path)) continue;
1444// CdWork();
74013535 1445 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 1446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 1447 else file = Form(format.Data(), irun);
0f389141 1448 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1449 if (fOverwriteMode) gGrid->Rm(file);
1450 else {
1451 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1452 continue;
1453 }
a8739e8a 1454 }
319593fb 1455 // If local collection file does not exist, create it via 'find' command.
3af861ff 1456 while (1) {
1457 ncount = 0;
1458 stage++;
1459 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1460 command = "find ";
1461 command += Form("%s -o %d ",options.Data(), nstart);
1462 command += path;
1463 command += pattern;
1464 command += conditions;
1465 TGridResult *res = gGrid->Command(command);
1466 if (res) delete res;
1467 // Write standard output to file
1468 gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1469 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1470 Bool_t nullFile = kFALSE;
1471 if (!hasGrep) {
1472 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1473 } else {
1474 nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1475 if (nullFile) {
1476 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1477 gSystem->Exec("rm -f __tmp*");
1604c9ed 1478 break;
3af861ff 1479 }
1480 TString line;
1481 ifstream in;
1482 in.open("__tmp__");
1483 in >> line;
1484 in.close();
1485 gSystem->Exec("rm -f __tmp__");
1486 ncount = line.Atoi();
1487 }
1488 nullResult = kFALSE;
1489 }
1490 if (ncount == gMaxEntries) {
1491 Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1492 if (fNrunsPerMaster > 1) {
1493 Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1494 file.Data(),gMaxEntries);
1495 return kFALSE;
1496 }
1497 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1498 if (!cbase) cbase = cadd;
1499 else {
1500 cbase->Add(cadd);
1501 delete cadd;
defd7a3a 1502 }
3af861ff 1503 nstart += ncount;
1504 } else {
c89a84bc 1505 if (cbase && fNrunsPerMaster<2) {
3af861ff 1506 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1507 printf("... please wait - TAlienCollection::Add() scales badly...\n");
1508 cbase->Add(cadd);
1509 delete cadd;
1510 cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1511 delete cbase; cbase = 0;
1512 } else {
1513 TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1514 }
1515 Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1516 break;
d3339be3 1517 }
319593fb 1518 }
a8739e8a 1519 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 1520 // Check if there is one run per master job.
1521 if (fNrunsPerMaster<2) {
1522 if (FileExists(file)) {
0f389141 1523 if (fOverwriteMode) gGrid->Rm(file);
1524 else {
1525 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1526 continue;
1527 }
319593fb 1528 }
1529 // Copy xml file to alien space
1530 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1531 if (!FileExists(file)) {
1532 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1533 return kFALSE;
1534 }
1535 } else {
1536 nruns++;
95e5b448 1537 // Check if the collection for the chunk exist locally.
1538 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 1539 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1540 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1541 else continue;
1542 }
84fcd93f 1543 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 1544 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 1545 schunk = Form(fRunPrefix.Data(), irun);
319593fb 1546 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1547 } else {
1548 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1549 cbase->Add(cadd);
1550 delete cadd;
1551 }
74013535 1552 format = Form("%%s_%s.xml", fRunPrefix.Data());
1553 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 1554 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 1555 continue;
1556 }
904f9f5f 1557 schunk = schunk2;
319593fb 1558 if (FileExists(schunk)) {
0f389141 1559 if (fOverwriteMode) gGrid->Rm(schunk);
1560 else {
1561 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1562 continue;
1563 }
319593fb 1564 }
84fcd93f 1565 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 1566 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 1567 if (FileExists(schunk)) {
0f389141 1568 if (fOverwriteMode) gGrid->Rm(schunk);
1569 else {
1570 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1571 continue;
1572 }
95e5b448 1573 }
319593fb 1574 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1575 if (!FileExists(schunk)) {
1576 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1577 return kFALSE;
1578 }
1579 }
c57f56b7 1580 }
a2f5fc01 1581 if (nullResult) {
d3339be3 1582 Error("CreateDataset", "No valid dataset corresponding to the query!");
1583 return kFALSE;
1584 }
a8739e8a 1585 }
c57f56b7 1586 return kTRUE;
1587}
1588
1589//______________________________________________________________________________
1590Bool_t AliAnalysisAlien::CreateJDL()
1591{
1592// Generate a JDL file according to current settings. The name of the file is
1593// specified by fJDLName.
ec5acdce 1594 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
c57f56b7 1595 Bool_t error = kFALSE;
1596 TObjArray *arr = 0;
1597 Bool_t copy = kTRUE;
d3b18c4c 1598 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 1599 Bool_t generate = kTRUE;
1600 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1601 if (!Connect()) {
1602 Error("CreateJDL", "Alien connection required");
1603 return kFALSE;
1604 }
1605 // Check validity of alien workspace
d3b18c4c 1606 TString workdir;
1607 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1608 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1609 workdir += fGridWorkingDir;
1610 if (generate) {
1611 TObjString *os;
1612 if (!fInputFiles) {
1613 Error("CreateJDL()", "Define some input files for your analysis.");
1614 error = kTRUE;
1615 }
1616 // Compose list of input files
1617 // Check if output files were defined
1618 if (!fOutputFiles.Length()) {
1619 Error("CreateJDL", "You must define at least one output file");
1620 error = kTRUE;
1621 }
1622 // Check if an output directory was defined and valid
1623 if (!fGridOutputDir.Length()) {
1624 Error("CreateJDL", "You must define AliEn output directory");
1625 error = kTRUE;
1626 } else {
d3b18c4c 1627 if (!fProductionMode) {
1628 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1629 if (!DirectoryExists(fGridOutputDir)) {
1630 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1631 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1632 } else {
1633 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1634 // error = kTRUE;
1635 }
67a3b254 1636 } else {
1637 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1638 }
1639 gGrid->Cd(workdir);
1640 }
c57f56b7 1641 }
1642 // Exit if any error up to now
1643 if (error) return kFALSE;
1644 // Set JDL fields
0f389141 1645 if (!fUser.IsNull()) {
1646 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1647 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1648 }
1649 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1650 TString mergeExec = fExecutable;
1651 mergeExec.ReplaceAll(".sh", "_merge.sh");
1652 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1653 mergeExec.ReplaceAll(".sh", ".C");
1654 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1655 if (!fArguments.IsNull())
1656 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1657 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1658 else {
1659 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1660 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1661 }
b3e07543 1662
1f0d1ca2 1663 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1664 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1665 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1666 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1667
0f389141 1668 if (fMaxInitFailed > 0) {
c57f56b7 1669 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1670 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1671 }
1672 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1673 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1674 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1675 }
1676 if (!IsOneStageMerging()) {
f47d5cb4 1677 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1678 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1679 }
1680 if (fSplitMode.Length()) {
c57f56b7 1681 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1682 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1683 }
1684 fMergingJDL->SetValue("Split", "\"se\"");
1685 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1686 if (!fAliROOTVersion.IsNull()) {
1687 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1688 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1689 }
1690 if (!fROOTVersion.IsNull()) {
c57f56b7 1691 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1692 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1693 }
1694 if (!fAPIVersion.IsNull()) {
c57f56b7 1695 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1696 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1697 }
648174cf 1698 if (!fExternalPackages.IsNull()) {
1699 arr = fExternalPackages.Tokenize(" ");
1700 TIter next(arr);
1701 while ((os=(TObjString*)next())) {
1702 TString pkgname = os->GetString();
1703 Int_t index = pkgname.Index("::");
1704 TString pkgversion = pkgname(index+2, pkgname.Length());
1705 pkgname.Remove(index);
1706 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1707 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1708 }
1709 delete arr;
1710 }
0f389141 1711 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1712 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1713 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1714 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1715 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1716 TString analysisFile = fExecutable;
1717 analysisFile.ReplaceAll(".sh", ".root");
1718 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1719 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1720 if (fAdditionalLibs.Length()) {
1721 arr = fAdditionalLibs.Tokenize(" ");
1722 TIter next(arr);
1723 while ((os=(TObjString*)next())) {
1724 if (os->GetString().Contains(".so")) continue;
1725 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1726 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1727 }
1728 delete arr;
1729 }
4e5c5506 1730 if (fPackages) {
1731 TIter next(fPackages);
1732 TObject *obj;
0f389141 1733 while ((obj=next())) {
4e5c5506 1734 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1735 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1736 }
4e5c5506 1737 }
95e16f01 1738 const char *comment = "List of output files and archives";
c57f56b7 1739 if (fOutputArchive.Length()) {
e4d9a069 1740 TString outputArchive = fOutputArchive;
1741 if (!fRegisterExcludes.IsNull()) {
1742 arr = fRegisterExcludes.Tokenize(" ");
1743 TIter next1(arr);
1744 while ((os=(TObjString*)next1())) {
1745 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1746 outputArchive.ReplaceAll(os->GetString(),"");
1747 }
1748 delete arr;
1749 }
1750 arr = outputArchive.Tokenize(" ");
c57f56b7 1751 TIter next(arr);
0f389141 1752 Bool_t first = kTRUE;
0f389141 1753 while ((os=(TObjString*)next())) {
0f389141 1754 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1755 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
0f389141 1756 else
95e16f01 1757 fGridJDL->AddToSet("Output", os->GetString());
1758 if (first) fGridJDL->AddToSetDescription("Output", comment);
0f389141 1759 first = kFALSE;
1760 }
c57f56b7 1761 delete arr;
f790bc1b 1762 // Output archive for the merging jdl
f790bc1b 1763 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1764 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1765 // Add normal output files, extra files + terminate files
1766 TString files = GetListOfFiles("outextter");
37d21c01 1767 // Do not register files in fRegisterExcludes
1768 if (!fRegisterExcludes.IsNull()) {
1769 arr = fRegisterExcludes.Tokenize(" ");
f790bc1b 1770 TIter next1(arr);
1771 while ((os=(TObjString*)next1())) {
1772 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1773 files.ReplaceAll(os->GetString(),"");
1774 }
66d71516 1775 delete arr;
0f389141 1776 }
f790bc1b 1777 files.ReplaceAll(".root", "*.root");
ec5acdce 1778
1779 if (mgr->IsCollectThroughput())
95e16f01 1780 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 1781 else
1782 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1783 } else {
95f84779 1784 TString files = fOutputArchive;
1785 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1786 outputArchive = files;
f790bc1b 1787 }
0f389141 1788 arr = outputArchive.Tokenize(" ");
705adb3e 1789 TIter next2(arr);
0f389141 1790 first = kTRUE;
705adb3e 1791 while ((os=(TObjString*)next2())) {
0cdf65a8 1792 TString currentfile = os->GetString();
0cdf65a8 1793 if (!currentfile.Contains("@") && fCloseSE.Length())
95e16f01 1794 fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
0f389141 1795 else
95e16f01 1796 fMergingJDL->AddToSet("Output", currentfile);
1797 if (first) fMergingJDL->AddToSetDescription("Output", comment);
0f389141 1798 first = kFALSE;
1799 }
1800 delete arr;
c57f56b7 1801 }
149d288c 1802 arr = fOutputFiles.Tokenize(",");
c57f56b7 1803 TIter next(arr);
0f389141 1804 Bool_t first = kTRUE;
43da816a 1805 while ((os=(TObjString*)next())) {
e1eaf596 1806 // Ignore ouputs in jdl that are also in outputarchive
1807 TString sout = os->GetString();
f790bc1b 1808 sout.ReplaceAll("*", "");
1809 sout.ReplaceAll(".root", "");
e1eaf596 1810 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1811 if (fOutputArchive.Contains(sout)) continue;
37d21c01 1812 // Ignore fRegisterExcludes
1813 if (fRegisterExcludes.Contains(sout)) continue;
0f389141 1814 if (!first) comment = NULL;
43da816a 1815 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1816 fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
43da816a 1817 else
95e16f01 1818 fGridJDL->AddToSet("Output", os->GetString());
1819 if (first) fGridJDL->AddToSetDescription("Output", comment);
f790bc1b 1820 if (fMergeExcludes.Contains(sout)) continue;
1821 if (!os->GetString().Contains("@") && fCloseSE.Length())
95e16f01 1822 fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
f790bc1b 1823 else
95e16f01 1824 fMergingJDL->AddToSet("Output", os->GetString());
1825 if (first) fMergingJDL->AddToSetDescription("Output", comment);
1826 first = kFALSE;
43da816a 1827 }
c57f56b7 1828 delete arr;
0f389141 1829 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1830 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1831 TString validationScript = fValidationScript;
0f389141 1832 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1833 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1834 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1835 if (fMasterResubmitThreshold) {
1836 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1837 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1838 }
d2a409b2 1839 // Write a jdl with 2 input parameters: collection name and output dir name.
1840 WriteJDL(copy);
c57f56b7 1841 }
1842 // Copy jdl to grid workspace
a8739e8a 1843 if (copy) {
b5e4aaa7 1844 // Check if an output directory was defined and valid
1845 if (!fGridOutputDir.Length()) {
1846 Error("CreateJDL", "You must define AliEn output directory");
1847 return kFALSE;
1848 } else {
1849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1850 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1851 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1852 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1853 } else {
1854 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1855 return kFALSE;
1856 }
1857 }
1858 gGrid->Cd(workdir);
1859 }
648174cf 1860 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1861 TString mergeJDLName = fExecutable;
1862 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1863 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1864 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1865 if (fProductionMode) {
648174cf 1866 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1867 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1868 }
648174cf 1869 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1870 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1871 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 1872 if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1873 Fatal("","Terminating");
1874// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1875 if (fMergeViaJDL) {
1876 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 1877// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1878 if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1879 Fatal("","Terminating");
0f389141 1880 }
648174cf 1881 }
c57f56b7 1882 if (fAdditionalLibs.Length()) {
e7c71df0 1883 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1884 TObjString *os;
1885 TIter next(arr);
1886 while ((os=(TObjString*)next())) {
c57f56b7 1887 if (os->GetString().Contains(".so")) continue;
4e5c5506 1888 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1889 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
6c395669 1890// TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1891 if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1892 Form("%s/%s", workdir.Data(), os->GetString().Data())))
1893 Fatal("","Terminating");
c57f56b7 1894 }
1895 delete arr;
1896 }
4e5c5506 1897 if (fPackages) {
1898 TIter next(fPackages);
1899 TObject *obj;
1900 while ((obj=next())) {
fdbbc7be 1901 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1902 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
6c395669 1903// TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1904 if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1905 Form("%s/%s", workdir.Data(), obj->GetName())))
1906 Fatal("","Terminating");
4e5c5506 1907 }
1908 }
c57f56b7 1909 }
1910 return kTRUE;
1911}
1912
a8739e8a 1913//______________________________________________________________________________
d2a409b2 1914Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1915{
1916// Writes one or more JDL's corresponding to findex. If findex is negative,
1917// all run numbers are considered in one go (jdl). For non-negative indices
1918// they correspond to the indices in the array fInputFiles.
1919 if (!fInputFiles) return kFALSE;
d3b18c4c 1920 TObject *os;
1921 TString workdir;
1922 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1923 workdir += fGridWorkingDir;
f47d5cb4 1924 TString stageName = "$2";
1925 if (fProductionMode) stageName = "$4";
1926 if (!fMergeDirName.IsNull()) {
1927 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1928 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1929 } else {
1930 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1931 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1932 }
d3b18c4c 1933 if (fProductionMode) {
a8739e8a 1934 TIter next(fInputFiles);
b3e07543 1935 while ((os=next())) {
d3b18c4c 1936 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1937 }
56754bf6 1938 if (!fOutputToRunNo)
1939 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1940 else
1941 fGridJDL->SetOutputDirectory(fGridOutputDir);
d3b18c4c 1942 } else {
1943 if (!fRunNumbers.Length() && !fRunRange[0]) {
1944 // One jdl with no parameters in case input data is specified by name.
1945 TIter next(fInputFiles);
1946 while ((os=next()))
1947 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1948 if (!fOutputSingle.IsNull())
1949 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1950 else {
1951 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1952 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1953 }
1954 } else {
1955 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1956 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1957 if (!fOutputSingle.IsNull()) {
1958 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1959 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1960 } else {
1961 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1962 }
1963 }
a8739e8a 1964 }
1965
a8739e8a 1966 // Generate the JDL as a string
1967 TString sjdl = fGridJDL->Generate();
0f389141 1968 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1969 // Final merge jdl
1970 if (!fMergeDirName.IsNull()) {
1971 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1972 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1973 } else {
1974 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1975 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1976 }
b3e07543 1977 TString sjdl2 = fMergingJDL->Generate();
1978 Int_t index, index1;
95e16f01 1979 sjdl.ReplaceAll("\",\"", "\",\n \"");
a8739e8a 1980 sjdl.ReplaceAll("(member", "\n (member");
1981 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1982 sjdl.ReplaceAll("{", "{\n ");
1983 sjdl.ReplaceAll("};", "\n};");
1984 sjdl.ReplaceAll("{\n \n", "{\n");
1985 sjdl.ReplaceAll("\n\n", "\n");
1986 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1987 sjdl1.ReplaceAll("\",\"", "\",\n \"");
0f389141 1988 sjdl1.ReplaceAll("(member", "\n (member");
1989 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1990 sjdl1.ReplaceAll("{", "{\n ");
1991 sjdl1.ReplaceAll("};", "\n};");
1992 sjdl1.ReplaceAll("{\n \n", "{\n");
1993 sjdl1.ReplaceAll("\n\n", "\n");
1994 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
95e16f01 1995 sjdl2.ReplaceAll("\",\"", "\",\n \"");
b3e07543 1996 sjdl2.ReplaceAll("(member", "\n (member");
1997 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1998 sjdl2.ReplaceAll("{", "{\n ");
1999 sjdl2.ReplaceAll("};", "\n};");
2000 sjdl2.ReplaceAll("{\n \n", "{\n");
2001 sjdl2.ReplaceAll("\n\n", "\n");
2002 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 2003 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 2004 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 2005 index = sjdl.Index("JDLVariables");
2006 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 2007 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2008 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 2009 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 2010 index = fJobTag.Index(":");
2011 if (index < 0) index = fJobTag.Length();
2012 TString jobTag = fJobTag;
f47d5cb4 2013 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 2014 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 2015 if (fProductionMode) {
2016 sjdl1.Prepend("# Generated merging jdl (production mode) \
2017 \n# $1 = full alien path to output directory to be merged \
2018 \n# $2 = train number \
2019 \n# $3 = production (like LHC10b) \
2020 \n# $4 = merging stage \
2021 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2022 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2023 sjdl2.Prepend("# Generated merging jdl \
2024 \n# $1 = full alien path to output directory to be merged \
2025 \n# $2 = train number \
2026 \n# $3 = production (like LHC10b) \
2027 \n# $4 = merging stage \
2028 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2029 } else {
2030 sjdl1.Prepend("# Generated merging jdl \
2031 \n# $1 = full alien path to output directory to be merged \
2032 \n# $2 = merging stage \
03335324 2033 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2034 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2035 sjdl2.Prepend("# Generated merging jdl \
2036 \n# $1 = full alien path to output directory to be merged \
2037 \n# $2 = merging stage \
03335324 2038 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 2039 }
0f389141 2040 index = sjdl1.Index("JDLVariables");
2041 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 2042 index = sjdl2.Index("JDLVariables");
2043 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 2044 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 2045 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2046 index = sjdl2.Index("Split =");
2047 if (index>=0) {
2048 index1 = sjdl2.Index("\n", index);
2049 sjdl2.Remove(index, index1-index+1);
2050 }
2051 index = sjdl2.Index("SplitMaxInputFileNumber");
2052 if (index>=0) {
2053 index1 = sjdl2.Index("\n", index);
2054 sjdl2.Remove(index, index1-index+1);
2055 }
2056 index = sjdl2.Index("InputDataCollection");
2057 if (index>=0) {
2058 index1 = sjdl2.Index(";", index);
2059 sjdl2.Remove(index, index1-index+1);
2060 }
2061 index = sjdl2.Index("InputDataListFormat");
2062 if (index>=0) {
2063 index1 = sjdl2.Index("\n", index);
2064 sjdl2.Remove(index, index1-index+1);
2065 }
2066 index = sjdl2.Index("InputDataList");
2067 if (index>=0) {
2068 index1 = sjdl2.Index("\n", index);
2069 sjdl2.Remove(index, index1-index+1);
2070 }
f47d5cb4 2071 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 2072 // Write jdl to file
a8739e8a 2073 ofstream out;
d2a409b2 2074 out.open(fJDLName.Data(), ios::out);
a8739e8a 2075 if (out.bad()) {
d3b18c4c 2076 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 2077 return kFALSE;
2078 }
2079 out << sjdl << endl;
b3e07543 2080 out.close();
0f389141 2081 TString mergeJDLName = fExecutable;
2082 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2083 if (fMergeViaJDL) {
2084 ofstream out1;
2085 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 2086 if (out1.bad()) {
d3b18c4c 2087 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 2088 return kFALSE;
2089 }
2090 out1 << sjdl1 << endl;
b3e07543 2091 out1.close();
2092 ofstream out2;
2093 TString finalJDL = mergeJDLName;
2094 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2095 out2.open(finalJDL.Data(), ios::out);
2096 if (out2.bad()) {
2097 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2098 return kFALSE;
2099 }
2100 out2 << sjdl2 << endl;
2101 out2.close();
0f389141 2102 }
a8739e8a 2103
2104 // Copy jdl to grid workspace
2105 if (!copy) {
d3b18c4c 2106 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 2107 } else {
d2a409b2 2108 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 2109 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 2110 TString finalJDL = mergeJDLName;
2111 finalJDL.ReplaceAll(".jdl", "_final.jdl");
2112 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 2113 if (fProductionMode) {
b5fe9cba 2114 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 2115 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 2116 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 2117 }
d2a409b2 2118 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 2119 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 2120 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 2121 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
6c395669 2122// TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2123 if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2124 Fatal("","Terminating");
0f389141 2125 if (fMergeViaJDL) {
b3e07543 2126 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
6c395669 2127// TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2128// TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2129 if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2130 Fatal("","Terminating");
2131 if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2132 Fatal("","Terminating");
0f389141 2133 }
a8739e8a 2134 }
2135 return kTRUE;
2136}
2137
c57f56b7 2138//______________________________________________________________________________
5513444a 2139Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 2140{
2141// Returns true if file exists.
5513444a 2142 if (!gGrid) return kFALSE;
2ea55496 2143 TString slfn = lfn;
2144 slfn.ReplaceAll("alien://","");
2145 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 2146 if (!res) return kFALSE;
2147 TMap *map = dynamic_cast<TMap*>(res->At(0));
2148 if (!map) {
2149 delete res;
2150 return kFALSE;
2151 }
2152 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2153 if (!objs || !objs->GetString().Length()) {
2154 delete res;
2155 return kFALSE;
2156 }
2157 delete res;
2158 return kTRUE;
2159}
2160
923e2ca5 2161//______________________________________________________________________________
2162Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2163{
2164// Returns true if directory exists. Can be also a path.
2165 if (!gGrid) return kFALSE;
2166 // Check if dirname is a path
2167 TString dirstripped = dirname;
2168 dirstripped = dirstripped.Strip();
2169 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
2170 TString dir = gSystem->BaseName(dirstripped);
2171 dir += "/";
2172 TString path = gSystem->DirName(dirstripped);
2173 TGridResult *res = gGrid->Ls(path, "-F");
2174 if (!res) return kFALSE;
2175 TIter next(res);
2176 TMap *map;
2177 TObject *obj;
2178 while ((map=dynamic_cast<TMap*>(next()))) {
2179 obj = map->GetValue("name");
2180 if (!obj) break;
2181 if (dir == obj->GetName()) {
2182 delete res;
2183 return kTRUE;
2184 }
2185 }
2186 delete res;
2187 return kFALSE;
2188}
2189
c57f56b7 2190//______________________________________________________________________________
a2f5fc01 2191void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 2192{
2193// Check input data type.
a2f5fc01 2194 isCollection = kFALSE;
2195 isXml = kFALSE;
2196 useTags = kFALSE;
c57f56b7 2197 if (!gGrid) {
2198 Error("CheckDataType", "No connection to grid");
2199 return;
2200 }
a2f5fc01 2201 isCollection = IsCollection(lfn);
c57f56b7 2202 TString msg = "\n##### file: ";
2203 msg += lfn;
a2f5fc01 2204 if (isCollection) {
c57f56b7 2205 msg += " type: raw_collection;";
2206 // special treatment for collections
a2f5fc01 2207 isXml = kFALSE;
c57f56b7 2208 // check for tag files in the collection
2209 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2210 if (!res) {
2211 msg += " using_tags: No (unknown)";
23329835 2212 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2213 return;
2214 }
2215 const char* typeStr = res->GetKey(0, "origLFN");
2216 if (!typeStr || !strlen(typeStr)) {
2217 msg += " using_tags: No (unknown)";
23329835 2218 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2219 return;
2220 }
2221 TString file = typeStr;
a2f5fc01 2222 useTags = file.Contains(".tag");
2223 if (useTags) msg += " using_tags: Yes";
c57f56b7 2224 else msg += " using_tags: No";
23329835 2225 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2226 return;
2227 }
2228 TString slfn(lfn);
2229 slfn.ToLower();
a2f5fc01 2230 isXml = slfn.Contains(".xml");
2231 if (isXml) {
c57f56b7 2232 // Open xml collection and check if there are tag files inside
2233 msg += " type: xml_collection;";
2234 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2235 if (!coll) {
2236 msg += " using_tags: No (unknown)";
23329835 2237 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2238 return;
2239 }
2240 TMap *map = coll->Next();
2241 if (!map) {
2242 msg += " using_tags: No (unknown)";
23329835 2243 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2244 return;
2245 }
2246 map = (TMap*)map->GetValue("");
2247 TString file;
2248 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 2249 useTags = file.Contains(".tag");
c57f56b7 2250 delete coll;
a2f5fc01 2251 if (useTags) msg += " using_tags: Yes";
c57f56b7 2252 else msg += " using_tags: No";
23329835 2253 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2254 return;
2255 }
a2f5fc01 2256 useTags = slfn.Contains(".tag");
c57f56b7 2257 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 2258 else msg += " type: unknown file;";
a2f5fc01 2259 if (useTags) msg += " using_tags: Yes";
c57f56b7 2260 else msg += " using_tags: No";
23329835 2261 Info("CheckDataType", "%s", msg.Data());
c57f56b7 2262}
2263
4e5c5506 2264//______________________________________________________________________________
2265void AliAnalysisAlien::EnablePackage(const char *package)
2266{
2267// Enables a par file supposed to exist in the current directory.
2268 TString pkg(package);
2269 pkg.ReplaceAll(".par", "");
2270 pkg += ".par";
2271 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 2272 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 2273 return;
2274 }
2275 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2276 Info("EnablePackage", "AliEn plugin will use .par packages");
2277 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2278 if (!fPackages) {
2279 fPackages = new TObjArray();
2280 fPackages->SetOwner();
2281 }
2282 fPackages->Add(new TObjString(pkg));
2283}
2284
3bdcb562 2285//______________________________________________________________________________
2286TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2287{
2288// Make a tree from files having the location specified in fFileForTestMode.
2289// Inspired from JF's CreateESDChain.
2290 if (fFileForTestMode.IsNull()) {
2291 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2292 return NULL;
2293 }
2294 if (gSystem->AccessPathName(fFileForTestMode)) {
2295 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2296 return NULL;
2297 }
2298 // Open the file
2299 ifstream in;
2300 in.open(fFileForTestMode);
2301 Int_t count = 0;
2302 // Read the input list of files and add them to the chain
b505f1f8 2303 TString line;
d6c60197 2304 TString streeName(treeName);
2305 if (IsUseMCchain()) streeName = "TE";
2306 TChain *chain = new TChain(streeName);
b505f1f8 2307 TChain *chainFriend = 0;
d6c60197 2308 if (!fFriendChainName.IsNull()) chainFriend = new TChain(streeName);
b505f1f8 2309 while (in.good())
2310 {
3bdcb562 2311 in >> line;
ffbe06b9 2312 if (line.IsNull() || line.BeginsWith("#")) continue;
3bdcb562 2313 if (count++ == fNtestFiles) break;
2314 TString esdFile(line);
2315 TFile *file = TFile::Open(esdFile);
b505f1f8 2316 if (file && !file->IsZombie()) {
2317 chain->Add(esdFile);
3bdcb562 2318 file->Close();
b505f1f8 2319 if (!fFriendChainName.IsNull()) {
4b5d9e21 2320 if (esdFile.Index("#") > -1)
2321 esdFile.Remove(esdFile.Index("#"));
2322 esdFile = gSystem->DirName(esdFile);
2323 esdFile += "/" + fFriendChainName;
b505f1f8 2324 file = TFile::Open(esdFile);
2325 if (file && !file->IsZombie()) {
2326 file->Close();
2327 chainFriend->Add(esdFile);
2328 } else {
2329 Fatal("GetChainForTestMode", "Cannot open friend file: %s", esdFile.Data());
2330 return 0;
2331 }
4b5d9e21 2332 }
3bdcb562 2333 } else {
2334 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2335 }
b505f1f8 2336 }
2337 in.close();
2338 if (!chain->GetListOfFiles()->GetEntries()) {
3bdcb562 2339 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2340 delete chain;
b505f1f8 2341 delete chainFriend;
3bdcb562 2342 return NULL;
b505f1f8 2343 }
3bdcb562 2344// chain->ls();
b505f1f8 2345 if (!fFriendChainName.IsNull()) chain->AddFriend(chainFriend);
2346 return chain;
3bdcb562 2347}
2348
d2a409b2 2349//______________________________________________________________________________
2350const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2351{
2352// Get job status for all jobs with jobid>jobidstart.
2353 static char mstatus[20];
2354 mstatus[0] = '\0';
2355 nrunning = 0;
2356 nwaiting = 0;
2357 nerror = 0;
2358 ndone = 0;
2359 TGridJobStatusList *list = gGrid->Ps("");
2360 if (!list) return mstatus;
2361 Int_t nentries = list->GetSize();
2362 TGridJobStatus *status;
2363 Int_t pid;
2364 for (Int_t ijob=0; ijob<nentries; ijob++) {
2365 status = (TGridJobStatus *)list->At(ijob);
8ddad121 2366 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 2367 if (pid<jobidstart) continue;
2368 if (pid == lastid) {
8ddad121 2369 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 2370 }
2371 switch (status->GetStatus()) {
2372 case TGridJobStatus::kWAITING:
2373 nwaiting++; break;
2374 case TGridJobStatus::kRUNNING:
2375 nrunning++; break;
2376 case TGridJobStatus::kABORTED:
2377 case TGridJobStatus::kFAIL:
2378 case TGridJobStatus::kUNKNOWN:
2379 nerror++; break;
2380 case TGridJobStatus::kDONE:
2381 ndone++;
2382 }
2383 }
2384 list->Delete();
2385 delete list;
2386 return mstatus;
2387}
2388
c57f56b7 2389//______________________________________________________________________________
2390Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2391{
2392// Returns true if file is a collection. Functionality duplicated from
2393// TAlien::Type() because we don't want to directly depend on TAlien.
2394 if (!gGrid) {
2395 Error("IsCollection", "No connection to grid");
2396 return kFALSE;
2397 }
2398 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2399 if (!res) return kFALSE;
2400 const char* typeStr = res->GetKey(0, "type");
2401 if (!typeStr || !strlen(typeStr)) return kFALSE;
2402 if (!strcmp(typeStr, "collection")) return kTRUE;
2403 delete res;
2404 return kFALSE;
2405}
2406
fe2d7fc2 2407//______________________________________________________________________________
2408Bool_t AliAnalysisAlien::IsSingleOutput() const
2409{
2410// Check if single-ouput option is on.
2411 return (!fOutputSingle.IsNull());
2412}
2413
16a4353c 2414//______________________________________________________________________________
2415void AliAnalysisAlien::Print(Option_t *) const
2416{
2417// Print current plugin settings.
84fcd93f 2418 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 2419 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2420 if (mgr && mgr->IsProofMode()) {
2421 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2422 if (TestBit(AliAnalysisGrid::kTest))
2423 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2424 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2425 if (!fProofDataSet.IsNull())
2426 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2427 if (fProofReset==1)
2428 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2429 if (fProofReset>1)
2430 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
874de7ea 2431 if (!fROOTVersion.IsNull())
2432 printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
d3b18c4c 2433 else
2434 printf("= ROOT version requested________________________ default\n");
2435 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2436 if (!fAliRootMode.IsNull())
2437 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2438 if (fNproofWorkers)
2439 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2440 if (fNproofWorkersPerSlave)
2441 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2442 if (TestSpecialBit(kClearPackages))
2443 printf("= ClearPackages requested...\n");
2444 if (fIncludePath.Data())
2445 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2446 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2447 if (fPackages && fPackages->GetEntries()) {
2448 TIter next(fPackages);
2449 TObject *obj;
2450 TString list;
2451 while ((obj=next())) list += obj->GetName();
2452 printf("= Par files to be used: ________________________ %s\n", list.Data());
2453 }
2454 if (TestSpecialBit(kProofConnectGrid))
2455 printf("= Requested PROOF connection to grid\n");
2456 return;
2457 }
e1c22e21 2458 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2459 if (fOverwriteMode) {
2460 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2461 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2462 }
348be253 2463 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2464 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 2465 printf("= Production mode:______________________________ %d\n", fProductionMode);
2466 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2467 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2468 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 2469 if (fUser.Length())
84fcd93f 2470 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2471 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2472 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2473 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
2474 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
2475 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 2476 if (fRunNumbers.Length())
84fcd93f 2477 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 2478 if (fRunRange[0])
74013535 2479 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 2480 if (!fRunRange[0] && !fRunNumbers.Length()) {
2481 TIter next(fInputFiles);
2482 TObject *obj;
2483 TString list;
2484 while ((obj=next())) list += obj->GetName();
84fcd93f 2485 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 2486 }
2487 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 2488 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2489 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2490 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2491 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
37d21c01 2492 printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
d3b18c4c 2493 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 2494 printf("=====================================================================\n");
2495 printf("= Job price: ___________________________________ %d\n", fPrice);
2496 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2497 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 2498 if (fMaxInitFailed>0)
84fcd93f 2499 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 2500 if (fMasterResubmitThreshold>0)
84fcd93f 2501 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 2502 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 2503 if (fNrunsPerMaster>0)
84fcd93f 2504 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2505 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 2506 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2507 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 2508 if (fArguments.Length())
84fcd93f 2509 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 2510 if (fExecutableArgs.Length())
2511 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 2512 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2513 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2514 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2515 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 2516 if (fDatasetName)
84fcd93f 2517 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2518 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 2519 if (fIncludePath.Data())
84fcd93f 2520 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 2521 if (fCloseSE.Length())
84fcd93f 2522 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 2523 if (fFriendChainName.Length())
84fcd93f 2524 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 2525 if (fPackages && fPackages->GetEntries()) {
16a4353c 2526 TIter next(fPackages);
2527 TObject *obj;
2528 TString list;
2529 while ((obj=next())) list += obj->GetName();
84fcd93f 2530 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 2531 }
2532}
2533
c57f56b7 2534//______________________________________________________________________________
2535void AliAnalysisAlien::SetDefaults()
2536{
2537// Set default values for everything. What cannot be filled will be left empty.
2538 if (fGridJDL) delete fGridJDL;
2539 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 2540 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 2541 fPrice = 1;
2542 fTTL = 30000;
2543 fSplitMaxInputFileNumber = 100;
2544 fMaxInitFailed = 0;
2545 fMasterResubmitThreshold = 0;
bb885a9e 2546 fNtestFiles = 10;
149d288c 2547 fNreplicas = 2;
a8739e8a 2548 fRunRange[0] = 0;
2549 fRunRange[1] = 0;
8e3bd06a 2550 fRunPrefix = "%d";
319593fb 2551 fNrunsPerMaster = 1;
16a4353c 2552 fMaxMergeFiles = 100;
c57f56b7 2553 fRunNumbers = "";
2554 fExecutable = "analysis.sh";
5aa94dc3 2555 fExecutableCommand = "root -b -q -x";
c57f56b7 2556 fArguments = "";
631c0b05 2557 fExecutableArgs = "";
c57f56b7 2558 fAnalysisMacro = "myAnalysis.C";
2559 fAnalysisSource = "";
2560 fAdditionalLibs = "";
2561 fSplitMode = "se";
2562 fAPIVersion = "";
2563 fROOTVersion = "";
2564 fAliROOTVersion = "";
2565 fUser = ""; // Your alien user name
2566 fGridWorkingDir = "";
2567 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2568 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 2569 fFriendChainName = "";
c57f56b7 2570 fGridOutputDir = "output";
b83f84f0 2571 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 2572 fOutputFiles = ""; // Like "AliAODs.root histos.root"
2573 fInputFormat = "xml-single";
2574 fJDLName = "analysis.jdl";
c6cb3634 2575 fJobTag = "Automatically generated analysis JDL";
bb885a9e 2576 fMergeExcludes = "";
0f389141 2577 fMergeViaJDL = 0;
348be253 2578 SetUseCopy(kTRUE);
2579 SetCheckCopy(kTRUE);
149d288c 2580 SetDefaultOutputs(kTRUE);
e1c22e21 2581 fOverwriteMode = 1;
c57f56b7 2582}
2583
a41b2882 2584//______________________________________________________________________________
2585void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2586{
2587 // Set file name for the chain of friends and optionally additional libs to be loaded.
2588 // Libs should be separated by blancs.
2589 fFriendChainName = name;
2590 fFriendLibs = libnames;
1bc06b63 2591 if (fFriendLibs.Length() && !fFriendLibs.Contains(".so")) {
a41b2882 2592 Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2593 }
2594}
2595
874de7ea 2596//______________________________________________________________________________
2597void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2598{
2599// Obsolete method. Use SetROOTVersion instead
2600 Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
95e16f01 2601 if (fROOTVersion.IsNull()) SetROOTVersion(version);
2602 else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
874de7ea 2603}
2604
0f389141 2605//______________________________________________________________________________
b3e07543 2606Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 2607{
b3e07543 2608// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2609 // First check if the result is already in the output directory.
2610 if (FileExists(Form("%s/%s",aliendir,filename))) {
2611 printf("Final merged results found. Not merging again.\n");
7c2cd90a 2612 return kFALSE;
2613 }
b3e07543 2614 // Now check the last stage done.
2615 Int_t stage = 0;
2616 while (1) {
2617 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2618 stage++;
2619 }
2620 // Next stage of merging
2621 stage++;
2622 TString pattern = "*root_archive.zip";
2623 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2624 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2625 if (res) delete res;
2626 // Write standard output to file
2627 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2628 // Count the number of files inside
2629 ifstream ifile;
2630 ifile.open(Form("Stage_%d.xml",stage));
2631 if (!ifile.good()) {
2632 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 2633 return kFALSE;
b3e07543 2634 }
2635 TString line;
2636 Int_t nfiles = 0;
2637 while (!ifile.eof()) {
2638 ifile >> line;
2639 if (line.Contains("/event")) nfiles++;
7c2cd90a 2640 }
b3e07543 2641 ifile.close();
2642 if (!nfiles) {
2643 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2644 return kFALSE;
2645 } else {
2646 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 2647 }
b3e07543 2648 // Copy the file in the output directory
2649 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
43d5e2dc 2650// TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
6c395669 2651 if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2652 Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
b3e07543 2653 // Check if this is the last stage to be done.
2654 Bool_t laststage = (nfiles<nperchunk);
2655 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
0ad3ea1f 2656 Int_t jobId = 0;
b3e07543 2657 if (laststage) {
2658 printf("### Submiting final merging stage %d\n", stage);
2659 TString finalJDL = jdl;
2660 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 2661 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
0ad3ea1f 2662 jobId = SubmitSingleJob(query);
b3e07543 2663 } else {
2664 printf("### Submiting merging stage %d\n", stage);
03335324 2665 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
0ad3ea1f 2666 jobId = SubmitSingleJob(query);
7c2cd90a 2667 }
0ad3ea1f 2668 if (!jobId) return kFALSE;
2669
2670 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2671 fGridJobIDs.Append(Form("%d", jobId));
2672 if (!fGridStages.IsNull()) fGridStages.Append(" ");
2673 fGridStages.Append(Form("%s_merge_stage%d",
2674 laststage ? "final" : "partial", stage));
2675
b3e07543 2676 return kTRUE;
2677}
7c2cd90a 2678
105631b5 2679//______________________________________________________________________________
2680AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2681{
2682// Loat the analysis manager from a file.
2683 TFile *file = TFile::Open(fname);
2684 if (!file) {
2685 ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2686 return 0;
2687 }
2688 TIter nextkey(file->GetListOfKeys());
2689 AliAnalysisManager *mgr = 0;
2690 TKey *key;
2691 while ((key=(TKey*)nextkey())) {
2692 if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2693 mgr = (AliAnalysisManager*)file->Get(key->GetName());
2694 }
2695 if (!mgr)
2696 ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2697 return mgr;
2698}
2699
7c2cd90a 2700//______________________________________________________________________________
2701Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2702{
2703// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2704 if (!gGrid) return 0;
2705 printf("=> %s ------> ",query);
2706 TGridResult *res = gGrid->Command(query);
2707 if (!res) return 0;
2708 TString jobId = res->GetKey(0,"jobId");
2709 delete res;
2710 if (jobId.IsNull()) {
2711 printf("submission failed. Reason:\n");
2712 gGrid->Stdout();
2713 gGrid->Stderr();
2714 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2715 return 0;
2716 }
0ad3ea1f 2717 Int_t ijobId = jobId.Atoi();
2718 printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2719 return ijobId;
7c2cd90a 2720}
2721
ec5acdce 2722//______________________________________________________________________________
2723Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2724{
2725// Merges a collection of output files using concatenation.
d66aa2a9 2726 TString scoll(collection);
2727 if (!scoll.Contains(".xml")) return kFALSE;
ec5acdce 2728 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2729 if (!coll) {
2730 ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2731 return kFALSE;
2732 }
2733 // Iterate grid collection
2734 TString outtmp;
2735 Bool_t merged = kFALSE;
2736 Int_t ifile = 0;
2737 while (coll->Next()) {
2738 TString fname = gSystem->DirName(coll->GetTURL());
2739 fname += "/";
2740 fname += output;
2741 outtmp = Form("%d_%s", ifile, output);
2742 if (!TFile::Cp(fname, outtmp)) {
2743 ::Error("MergeInfo", "Could not copy %s", fname.Data());
2744 continue;
2745 }
2746 ifile++;
2747 if (ifile<2) {
2748 gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2749 continue;
2750 }
2751 gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2752 gSystem->Exec("cp tempmerged lastmerged");
2753 }
2754 if (ifile) {
2755 gSystem->Exec(Form("cp lastmerged %s", output));
2756 gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2757 merged = kTRUE;
2758 }
2759 return merged;
2760}
2761
7c2cd90a 2762//______________________________________________________________________________
b3e07543 2763Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2764{
b3e07543 2765// Merge given output files from basedir. Basedir can be an alien output directory
2766// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2767// files in a group (ignored for xml input). Merging can be done in stages:
2768// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2769// stage=1 : works with an xml of all root_archive.zip in the output directory
2770// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2771 TString outputFile = output;
0f389141 2772 TString command;
a2f5fc01 2773 TString outputChunk;
2774 TString previousChunk = "";
b3e07543 2775 TObjArray *listoffiles = new TObjArray();
2776// listoffiles->SetOwner();
a2f5fc01 2777 Int_t countChunk = 0;
2778 Int_t countZero = nmaxmerge;
0f389141 2779 Bool_t merged = kTRUE;
d66aa2a9 2780 Bool_t isGrid = kTRUE;
a2f5fc01 2781 Int_t index = outputFile.Index("@");
2782 if (index > 0) outputFile.Remove(index);
7c2cd90a 2783 TString inputFile = outputFile;
b3e07543 2784 TString sbasedir = basedir;
2785 if (sbasedir.Contains(".xml")) {
2786 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2787 nmaxmerge = 9999999;
2788 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2789 if (!coll) {
2790 ::Error("MergeOutput", "Input XML collection empty.");
2791 return kFALSE;
2792 }
2793 // Iterate grid collection
2794 while (coll->Next()) {
2795 TString fname = gSystem->DirName(coll->GetTURL());
2796 fname += "/";
2797 fname += inputFile;
2798 listoffiles->Add(new TNamed(fname.Data(),""));
2799 }
ffbe06b9 2800 } else if (sbasedir.Contains(".txt")) {
d66aa2a9 2801 // The file having the .txt extension is expected to contain a list of
2802 // folders where the output files will be looked. For alien folders,
2803 // the full folder LFN is expected (starting with alien://)
ffbe06b9 2804 // Assume lfn's on each line
2805 TString line;
2806 ifstream in;
2807 in.open(sbasedir);
d66aa2a9 2808 if (in.fail()) {
2809 ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2810 return kTRUE;
2811 }
ffbe06b9 2812 Int_t nfiles = 0;
2813 while (in.good()) {
2814 in >> line;
2815 if (line.IsNull() || line.BeginsWith("#")) continue;
d66aa2a9 2816 line.Strip();
2817 if (!line.Contains("alien:")) isGrid = kFALSE;
2818 line += "/";
2819 line += outputFile;
ffbe06b9 2820 nfiles++;
2821 listoffiles->Add(new TNamed(line.Data(),""));
2822 }
d66aa2a9 2823 in.close();
ffbe06b9 2824 if (!nfiles) {
2825 ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2826 delete listoffiles;
2827 return kFALSE;
2828 }
b3e07543 2829 } else {
2830 command = Form("find %s/ *%s", basedir, inputFile.Data());
2831 printf("command: %s\n", command.Data());
2832 TGridResult *res = gGrid->Command(command);
2833 if (!res) {
2834 ::Error("MergeOutput","No result for the find command\n");
2835 delete listoffiles;
2836 return kFALSE;
2837 }
2838 TIter nextmap(res);
2839 TMap *map = 0;
2840 while ((map=(TMap*)nextmap())) {
2841 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2842 if (!objs || !objs->GetString().Length()) {
2843 // Nothing found - skip this output
2844 delete res;
2845 delete listoffiles;
2846 return kFALSE;
2847 }
2848 listoffiles->Add(new TNamed(objs->GetName(),""));
2849 }
2850 delete res;
2851 }
2852 if (!listoffiles->GetEntries()) {
7c2cd90a 2853 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2854 delete listoffiles;
0f389141 2855 return kFALSE;
2856 }
2857
2858 TFileMerger *fm = 0;
b3e07543 2859 TIter next0(listoffiles);
2860 TObjArray *listoffilestmp = new TObjArray();
2861 listoffilestmp->SetOwner();
2862 TObject *nextfile;
2863 TString snextfile;
2864 // Keep only the files at upper level
2865 Int_t countChar = 0;
2866 while ((nextfile=next0())) {
2867 snextfile = nextfile->GetName();
2868 Int_t crtCount = snextfile.CountChar('/');
2869 if (nextfile == listoffiles->First()) countChar = crtCount;
2870 if (crtCount < countChar) countChar = crtCount;
2871 }
2872 next0.Reset();
2873 while ((nextfile=next0())) {
2874 snextfile = nextfile->GetName();
2875 Int_t crtCount = snextfile.CountChar('/');
2876 if (crtCount > countChar) {
2877 delete nextfile;
2878 continue;
2879 }
2880 listoffilestmp->Add(nextfile);
2881 }
2882 delete listoffiles;
2883 listoffiles = listoffilestmp; // Now contains 'good' files
2884 listoffiles->Print();
2885 TIter next(listoffiles);
7c2cd90a 2886 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2887 outputChunk = outputFile;
2888 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2889 // Check for existent temporary merge files
e1c22e21 2890 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2891 // Preserve old merging functionality for stage 0.
2892 if (stage==0) {
2893 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2894 while (1) {
2895 // Skip as many input files as in a chunk
2ea55496 2896 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2897 nextfile = next();
2898 if (!nextfile) {
2ea55496 2899 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2900 delete listoffiles;
2ea55496 2901 return kFALSE;
2902 }
b3e07543 2903 snextfile = nextfile->GetName();
7c2cd90a 2904 }
2905 outputChunk = outputFile;
2906 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2907 countChunk++;
2908 if (gSystem->AccessPathName(outputChunk)) continue;
2909 // Merged file with chunks up to <countChunk> found
2910 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2911 previousChunk = outputChunk;
2912 break;
2913 }
2914 }
2915 countZero = nmaxmerge;
2916
b3e07543 2917 while ((nextfile=next())) {
2918 snextfile = nextfile->GetName();
2ea55496 2919 // Loop 'find' results and get next LFN
7c2cd90a 2920 if (countZero == nmaxmerge) {
2921 // First file in chunk - create file merger and add previous chunk if any.
d66aa2a9 2922 fm = new TFileMerger(isGrid);
7c2cd90a 2923 fm->SetFastMethod(kTRUE);
2924 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2925 outputChunk = outputFile;
2926 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2927 }
2928 // If last file found, put merged results in the output file
b3e07543 2929 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2930 // Add file to be merged and decrement chunk counter.
b3e07543 2931 fm->AddFile(snextfile);
7c2cd90a 2932 countZero--;
b3e07543 2933 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2934 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2935 // Nothing found - skip this output
2936 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2937 merged = kFALSE;
2938 break;
7c2cd90a 2939 }
2940 fm->OutputFile(outputChunk);
2941 // Merge the outputs, then go to next chunk
2942 if (!fm->Merge()) {
2943 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2944 merged = kFALSE;
2945 break;
7c2cd90a 2946 } else {
2947 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2948 gSystem->Unlink(previousChunk);
2949 }
b3e07543 2950 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2951 countChunk++;
2952 countZero = nmaxmerge;
2953 previousChunk = outputChunk;
0f389141 2954 }
0f389141 2955 }
b3e07543 2956 delete listoffiles;
2957 delete fm;
7c2cd90a 2958 return merged;
2959 }
2960 // Merging stage different than 0.
2961 // Move to the begining of the requested chunk.
d66aa2a9 2962 fm = new TFileMerger(isGrid);
7c2cd90a 2963 fm->SetFastMethod(kTRUE);
b3e07543 2964 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2965 delete listoffiles;
7c2cd90a 2966 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2967 // Nothing found - skip this output
2968 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2969 delete fm;
2970 return kFALSE;
2971 }
b3e07543 2972 fm->OutputFile(outputFile);
7c2cd90a 2973 // Merge the outputs
2974 if (!fm->Merge()) {
2975 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2976 delete fm;
2977 return kFALSE;
2978 } else {
b3e07543 2979 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2980 }
7c2cd90a 2981 delete fm;
2982 return kTRUE;
0f389141 2983}
2984
c57f56b7 2985//______________________________________________________________________________
2986Bool_t AliAnalysisAlien::MergeOutputs()
2987{
2988// Merge analysis outputs existing in the AliEn space.
2989 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2990 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2991 if (!Connect()) {
2992 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2993 return kFALSE;
0f389141 2994 }
c9e8f7fd 2995 if (fMergeViaJDL) {
2996 if (!TestBit(AliAnalysisGrid::kMerge)) {
2997 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2998 return kFALSE;
2999 }
3000 if (fProductionMode) {
3001 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3002 return kFALSE;
3003 }
0f389141 3004 Info("MergeOutputs", "Submitting merging JDL");
a03be957 3005 if (!SubmitMerging()) return kFALSE;
0f389141 3006 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 3007 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 3008 return kFALSE;
c57f56b7 3009 }
3010 // Get the output path
dc030a24 3011 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 3012 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 3013 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3014 return kFALSE;
3015 }
3016 if (!fOutputFiles.Length()) {
3017 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3018 return kFALSE;
0f389141 3019 }
3020 // Check if fast read option was requested
e1c22e21 3021 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3022 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 3023 if (fFastReadOption) {
d3339be3 3024 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3025 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 3026 gEnv->SetValue("XNet.ConnectTimeout",50);
3027 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 3028 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 3029 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 3030 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 3031 }
e8b839ab 3032 // Make sure we change the temporary directory
3033 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 3034 // Set temporary compilation directory to current one
3035 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 3036 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3037 TIter next(list);
3038 TObjString *str;
a2f5fc01 3039 TString outputFile;
c57f56b7 3040 Bool_t merged = kTRUE;
3041 while((str=(TObjString*)next())) {
a2f5fc01 3042 outputFile = str->GetString();
3043 Int_t index = outputFile.Index("@");
3044 if (index > 0) outputFile.Remove(index);
3045 TString outputChunk = outputFile;
3046 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 3047 // Skip already merged outputs
a2f5fc01 3048 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 3049 if (fOverwriteMode) {
a2f5fc01 3050 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3051 gSystem->Unlink(outputFile);
3052 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3053 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3054 outputChunk.Data());
3055 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3056 }
3057 } else {
a2f5fc01 3058 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 3059 continue;
3060 }
3061 } else {
a2f5fc01 3062 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 3063 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 3064 outputChunk.Data());
3065 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 3066 }
3067 }
37d21c01 3068 if (fMergeExcludes.Contains(outputFile.Data()) ||
3069 fRegisterExcludes.Contains(outputFile.Data())) continue;
16a4353c 3070 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 3071 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 3072 if (!merged) {
3073 Error("MergeOutputs", "Terminate() will NOT be executed");
09d5920f 3074 delete list;
3075 return kFALSE;
ff07ec61 3076 }
3077 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3078 if (fileOpened) fileOpened->Close();
c57f56b7 3079 }
09d5920f 3080 delete list;
0f389141 3081 return kTRUE;
c57f56b7 3082}
3083
bb885a9e 3084//______________________________________________________________________________
3085void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3086{
3087// Use the output files connected to output containers from the analysis manager
3088// rather than the files defined by SetOutputFiles
3089 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 3090 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 3091 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3092}
3093
149d288c 3094//______________________________________________________________________________
3095void AliAnalysisAlien::SetOutputFiles(const char *list)
3096{
3097// Manually set the output files list.
3098// Removes duplicates. Not allowed if default outputs are not disabled.
3099 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3100 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3101 return;
3102 }
3103 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3104 fOutputFiles = "";
3105 TString slist = list;
3106 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3107 TObjArray *arr = slist.Tokenize(" ");
3108 TObjString *os;
3109 TIter next(arr);
3110 TString sout;
3111 while ((os=(TObjString*)next())) {
3112 sout = os->GetString();
3113 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3114 if (fOutputFiles.Contains(sout)) continue;
3115 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3116 fOutputFiles += sout;
3117 }
3118 delete arr;
f790bc1b 3119}
149d288c 3120
3121//______________________________________________________________________________
3122void AliAnalysisAlien::SetOutputArchive(const char *list)
3123{
3124// Manually set the output archive list. Free text - you are on your own...
3125// Not allowed if default outputs are not disabled.
3126 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3127 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3128 return;
3129 }
3130 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3131 fOutputArchive = list;
3132}
3133
3134//______________________________________________________________________________
3135void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
3136{
3137// Setting a prefered output SE is not allowed anymore.
3138 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
3139}
3140
28ce6c9c 3141//______________________________________________________________________________
3142void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3143{
3144// Set some PROOF special parameter.
3145 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3146 if (pair) {
3147 TObject *old = pair->Key();
3148 TObject *val = pair->Value();
3149 fProofParam.Remove(old);
3150 delete old;
3151 delete val;
3152 }
3153 fProofParam.Add(new TObjString(pname), new TObjString(value));
3154}
3155
3156//______________________________________________________________________________
3157const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3158{
3159// Returns a special PROOF parameter.
3160 TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3161 if (!pair) return 0;
3162 return pair->Value()->GetName();
3163}
3164
5513444a 3165//______________________________________________________________________________
3166Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 3167{
3168// Start remote grid analysis.
3bdcb562 3169 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3170 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3171 if (!mgr || !mgr->IsInitialized()) {
3172 Error("StartAnalysis", "You need an initialized analysis manager for this");
3173 return kFALSE;
3174 }
3175 // Are we in PROOF mode ?
3176 if (mgr->IsProofMode()) {
baa34407 3177 if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3178 else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 3179 if (fProofCluster.IsNull()) {
3180 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3181 return kFALSE;
3182 }
3183 if (fProofDataSet.IsNull() && !testMode) {
3184 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3185 return kFALSE;
3186 }
3187 // Set the needed environment
3188 gEnv->SetValue("XSec.GSI.DelegProxy","2");
3189 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3190 if (fProofReset && !testMode) {
3191 if (fProofReset==1) {
3192 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3193 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3194 } else {
3195 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3196 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3197 }
3198 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3199 return kFALSE;
3200 }
baa34407 3201
3202 if (!testMode) {
3203 // Check if there is an old active session
3204 Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3205 if (nsessions) {
3206 Error("StartAnalysis","You have to reset your old session first\n");
3207 return kFALSE;
3208 }
53911ca6 3209 }
3bdcb562 3210 // Do we need to change the ROOT version ? The success of this cannot be checked.
874de7ea 3211 if (!fROOTVersion.IsNull() && !testMode) {
cbccef47 3212 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
874de7ea 3213 fProofCluster.Data(), fROOTVersion.Data()));
3bdcb562 3214 }
3215 // Connect to PROOF and check the status
3216 Long_t proof = 0;
27734f0e 3217 TString sworkers;
3218 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3219 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 3220 if (!testMode) {
27734f0e 3221 if (!sworkers.IsNull())
3222 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 3223 else
3224 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3225 } else {
3226 proof = gROOT->ProcessLine("TProof::Open(\"\");");
3227 if (!proof) {
3228 Error("StartAnalysis", "Could not start PROOF in test mode");
3229 return kFALSE;
3230 }
3231 }
3232 if (!proof) {
3233 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3234 return kFALSE;
3235 }
27734f0e 3236 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3237 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
28ce6c9c 3238 // Set proof special parameters if any
3239 TIter nextpp(&fProofParam);
3240 TObject *proofparam;
3241 while ((proofparam=nextpp())) {
3242 TString svalue = GetProofParameter(proofparam->GetName());
3243 gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3244 }
3bdcb562 3245 // Is dataset existing ?
3246 if (!testMode) {
3247 TString dataset = fProofDataSet;
3248 Int_t index = dataset.Index("#");
3249 if (index>=0) dataset.Remove(index);
830acc4c 3250// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3251// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3252// return kFALSE;
3253// }
3254// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 3255 }
3256 // Is ClearPackages() needed ?
3257 if (TestSpecialBit(kClearPackages)) {
3258 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3259 gROOT->ProcessLine("gProof->ClearPackages();");
3260 }
3261 // Is a given aliroot mode requested ?
3262 TList optionsList;
1ac846e9 3263 TString parLibs;
830acc4c 3264 if (!fAliRootMode.IsNull()) {
3bdcb562 3265 TString alirootMode = fAliRootMode;
3266 if (alirootMode == "default") alirootMode = "";
3267 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3268 optionsList.SetOwner();
3269 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3270 // Check the additional libs to be loaded
3271 TString extraLibs;
1ac846e9 3272 Bool_t parMode = kFALSE;
dc1ffbcb 3273 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:OADB:ANALYSISalice";
3bdcb562 3274 // Parse the extra libs for .so
3275 if (fAdditionalLibs.Length()) {
a41b2882 3276 TString additionalLibs = fAdditionalLibs;
3277 additionalLibs.Strip();
3278 if (additionalLibs.Length() && fFriendLibs.Length())
3279 additionalLibs += " ";
3280 additionalLibs += fFriendLibs;
3281 TObjArray *list = additionalLibs.Tokenize(" ");
3bdcb562 3282 TIter next(list);
3283 TObjString *str;
1ac846e9 3284 while((str=(TObjString*)next())) {
3285 if (str->GetString().Contains(".so")) {
3286 if (parMode) {
3287 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3288 break;
3289 }
3290 TString stmp = str->GetName();
3291 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3292 stmp.ReplaceAll(".so","");
3293 if (!extraLibs.IsNull()) extraLibs += ":";
3294 extraLibs += stmp;
3295 continue;
3296 }
3297 if (str->GetString().Contains(".par")) {
3298 // The first par file found in the list will not allow any further .so
3299 parMode = kTRUE;
3300 if (!parLibs.IsNull()) parLibs += ":";
3301 parLibs += str->GetName();
3302 continue;
3303 }
3bdcb562 3304 }
3305 if (list) delete list;
3306 }
a41b2882 3307 if (!extraLibs.IsNull()) {
3308 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3309 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3310 }
830acc4c 3311 // Check extra includes
3312 if (!fIncludePath.IsNull()) {
3313 TString includePath = fIncludePath;
3314 includePath.ReplaceAll(" ",":");
dc030a24 3315 includePath.ReplaceAll("$ALICE_ROOT/","");
3316 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 3317 includePath.ReplaceAll("-I","");
dc030a24 3318 includePath.Remove(TString::kTrailing, ':');
830acc4c 3319 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3320 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3321 }
27734f0e 3322 // Check if connection to grid is requested
3323 if (TestSpecialBit(kProofConnectGrid))
3324 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 3325 // Enable AliRoot par
3326 if (testMode) {
3327 // Enable proof lite package
3328 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3329 for (Int_t i=0; i<optionsList.GetSize(); i++) {
3330 TNamed *obj = (TNamed*)optionsList.At(i);
3331 printf("%s %s\n", obj->GetName(), obj->GetTitle());
3332 }
3333 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 3334 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 3335 Info("StartAnalysis", "AliRootProofLite enabled");
3336 } else {
3337 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3338 return kFALSE;
3339 }
3340 } else {
dc030a24 3341 if ( ! fAliROOTVersion.IsNull() ) {
3342 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3343 fAliROOTVersion.Data(), &optionsList))) {
3344 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3345 return kFALSE;
3346 }
3347 }
27734f0e 3348 }
1ac846e9 3349 // Enable first par files from fAdditionalLibs
3350 if (!parLibs.IsNull()) {
3351 TObjArray *list = parLibs.Tokenize(":");
3352 TIter next(list);
3353 TObjString *package;
3354 while((package=(TObjString*)next())) {
40501bb7 3355 TString spkg = package->GetName();
3356 spkg.ReplaceAll(".par", "");
3357 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 3358 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 3359 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3360 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 3361 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3362 return kFALSE;
3363 }
3364 } else {
3365 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3366 return kFALSE;
3367 }
3368 }
3369 if (list) delete list;
3370 }
3bdcb562 3371 } else {
3372 if (fAdditionalLibs.Contains(".so") && !testMode) {
3373 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3374 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3375 return kFALSE;
3376 }
3377 }
3378 // Enable par files if requested
3379 if (fPackages && fPackages->GetEntries()) {
3380 TIter next(fPackages);
3381 TObject *package;
3382 while ((package=next())) {
1ac846e9 3383 // Skip packages already enabled
3384 if (parLibs.Contains(package->GetName())) continue;
40501bb7 3385 TString spkg = package->GetName();
3386 spkg.ReplaceAll(".par", "");
3387 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 3388 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 3389 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 3390 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3391 return kFALSE;
3392 }
3393 } else {
3394 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3395 return kFALSE;
3396 }
3397 }
3398 }
3399 // Do we need to load analysis source files ?
3400 // NOTE: don't load on client since this is anyway done by the user to attach his task.
3401 if (fAnalysisSource.Length()) {
3402 TObjArray *list = fAnalysisSource.Tokenize(" ");
3403 TIter next(list);
3404 TObjString *str;
3405 while((str=(TObjString*)next())) {
3406 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 3407 }
3bdcb562 3408 if (list) delete list;
3409 }
830acc4c 3410 if (testMode) {
3411 // Register dataset to proof lite.
3412 if (fFileForTestMode.IsNull()) {
3413 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3414 return kFALSE;
3415 }
3416 if (gSystem->AccessPathName(fFileForTestMode)) {
3417 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3418 return kFALSE;
3419 }
3420 TFileCollection *coll = new TFileCollection();
3421 coll->AddFromFile(fFileForTestMode);
8ddad121 3422 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 3423 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 3424 }
3bdcb562 3425 return kTRUE;
3426 }
c57f56b7 3427
43da816a 3428 // Check if output files have to be taken from the analysis manager
3429 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 3430 // Add output files and AOD files
3431 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 3432 // Add extra files registered to the analysis manager
f790bc1b 3433 TString extra = GetListOfFiles("ext");
3434 if (!extra.IsNull()) {
149d288c 3435 extra.ReplaceAll(".root", "*.root");
f790bc1b 3436 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 3437 fOutputFiles += extra;
84fcd93f 3438 }
149d288c 3439 // Compose the output archive.
b83f84f0 3440 fOutputArchive = "log_archive.zip:std*@disk=1 ";
ec5acdce 3441 if (mgr->IsCollectThroughput())
95e16f01 3442 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
ec5acdce 3443 else
3444 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 3445 }
f7b1cbc2 3446// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 3447 if (TestBit(AliAnalysisGrid::kOffline)) {
3448 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3449 \n there nor any job run. You can revise the JDL and analysis \
3450 \n macro then run the same in \"submit\" mode.");
3451 } else if (TestBit(AliAnalysisGrid::kTest)) {
3452 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3453 \n dataset.");
3454 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3455 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3456 \n space and job submitted.");
3457 } else if (TestBit(AliAnalysisGrid::kMerge)) {
3458 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 3459 if (fMergeViaJDL) CheckInputData();
5513444a 3460 return kTRUE;
c57f56b7 3461 } else {
3462 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3463 }
3464
348be253 3465 Print();
c57f56b7 3466 if (!Connect()) {
3467 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 3468 return kFALSE;
16a4353c 3469 }
d3b18c4c 3470 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 3471 if (!CheckInputData()) {
3472 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 3473 return kFALSE;
c57f56b7 3474 }
d3339be3 3475 if (!CreateDataset(fDataPattern)) {
3476 TString serror;
3477 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3478 if (fRunNumbers.Length()) serror = "run numbers";
3479 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3480 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3481 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3482 return kFALSE;
3483 }
21ca8e59 3484 WriteAnalysisFile();
c57f56b7 3485 WriteAnalysisMacro();
3486 WriteExecutable();
3487 WriteValidationScript();
0f389141 3488 if (fMergeViaJDL) {
3489 WriteMergingMacro();
3490 WriteMergeExecutable();
3491 WriteValidationScript(kTRUE);
3492 }
5513444a 3493 if (!CreateJDL()) return kFALSE;
3494 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 3495 if (testMode) {
c57f56b7 3496 // Locally testing the analysis
3497 Info("StartAnalysis", "\n_______________________________________________________________________ \
3498 \n Running analysis script in a daughter shell as on a worker node \
3499 \n_______________________________________________________________________");
149d288c 3500 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 3501 TIter next(list);
3502 TObjString *str;
a2f5fc01 3503 TString outputFile;
c57f56b7 3504 while((str=(TObjString*)next())) {
a2f5fc01 3505 outputFile = str->GetString();
3506 Int_t index = outputFile.Index("@");
3507 if (index > 0) outputFile.Remove(index);
3508 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 3509 }
3510 delete list;
3511 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 3512 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 3513// gSystem->Exec("cat stdout");
5513444a 3514 return kFALSE;
c57f56b7 3515 }
5513444a 3516 // Check if submitting is managed by LPM manager
a3e84053 3517 if (fProductionMode) {
7ac61826 3518 //TString prodfile = fJDLName;
3519 //prodfile.ReplaceAll(".jdl", ".prod");
3520 //WriteProductionFile(prodfile);
5513444a 3521 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3522 return kFALSE;
3523 }
a8739e8a 3524 // Submit AliEn job(s)
d2a409b2 3525 gGrid->Cd(fGridOutputDir);
a8739e8a 3526 TGridResult *res;
c57f56b7 3527 TString jobID = "";
0ad3ea1f 3528 fGridJobIDs = "";
3529 fGridStages = "";
d2a409b2 3530 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 3531 // Submit a given xml or a set of runs
a8739e8a 3532 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 3533 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 3534 if (res) {
3535 const char *cjobId = res->GetKey(0,"jobId");
3536 if (!cjobId) {
a03be957 3537 gGrid->Stdout();
3538 gGrid->Stderr();
a8739e8a 3539 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 3540 return kFALSE;
a8739e8a 3541 } else {
3542 Info("StartAnalysis", "\n_______________________________________________________________________ \
3543 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3544 \n_______________________________________________________________________",
3545 fJDLName.Data(), cjobId);
3546 jobID = cjobId;
0ad3ea1f 3547 if (jobID.Atoi()) {
3548 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3549 fGridJobIDs.Append(jobID);
3550 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3551 fGridStages.Append("full");
3552 }
a8739e8a 3553 }
3554 delete res;
a03be957 3555 } else {
3556 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3557 return kFALSE;
a8739e8a 3558 }
3559 } else {
d2a409b2 3560 // Submit for a range of enumeration of runs.
a03be957 3561 if (!Submit()) return kFALSE;
0ad3ea1f 3562 jobID = fGridJobIDs;
c57f56b7 3563 }
a8739e8a 3564
d88e4159 3565 if (fDropToShell) {
3566 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3567 \n You may exit at any time and terminate the job later using the option <terminate> \
3568 \n ##################################################################################", jobID.Data());
3569 gSystem->Exec("aliensh");
3570 } else {
3571 Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3572 \n Remember to terminate the job later using the option <terminate> \
3573 \n ##################################################################################", jobID.Data());
3574 }
5513444a 3575 return kTRUE;
c57f56b7 3576}
3577
f790bc1b 3578//______________________________________________________________________________
3579const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3580{
3581// Get a comma-separated list of output files of the requested type.
3582// Type can be (case unsensitive):
3583// aod - list of aod files (std, extensions and filters)
3584// out - list of output files connected to containers (but not aod's or extras)
3585// ext - list of extra files registered to the manager
3586// ter - list of files produced in terminate
3587 static TString files;
3588 files = "";
3589 TString stype = type;
3590 stype.ToLower();
3591 TString aodfiles, extra;
3592 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3593 if (!mgr) {
3594 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3595 return files.Data();
3596 }
3597 if (mgr->GetOutputEventHandler()) {
3598 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3599 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
3600 if (!extraaod.IsNull()) {
3601 aodfiles += ",";
3602 aodfiles += extraaod;
3603 }
3604 }
3605 if (stype.Contains("aod")) {
3606 files = aodfiles;
3607 if (stype == "aod") return files.Data();
3608 }
3609 // Add output files that are not in the list of AOD files
3610 TString outputfiles = "";
3611 TIter next(mgr->GetOutputs());
3612 AliAnalysisDataContainer *output;
3613 const char *filename = 0;
3614 while ((output=(AliAnalysisDataContainer*)next())) {
3615 filename = output->GetFileName();
3616 if (!(strcmp(filename, "default"))) continue;
3617 if (outputfiles.Contains(filename)) continue;
3618 if (aodfiles.Contains(filename)) continue;
3619 if (!outputfiles.IsNull()) outputfiles += ",";
3620 outputfiles += filename;
3621 }
3622 if (stype.Contains("out")) {
3623 if (!files.IsNull()) files += ",";
3624 files += outputfiles;
3625 if (stype == "out") return files.Data();
3626 }
3627 // Add extra files registered to the analysis manager
3628 TString sextra;
3629 extra = mgr->GetExtraFiles();
3630 if (!extra.IsNull()) {
3631 extra.Strip();
3632 extra.ReplaceAll(" ", ",");
3633 TObjArray *fextra = extra.Tokenize(",");
3634 TIter nextx(fextra);
3635 TObject *obj;
3636 while ((obj=nextx())) {
3637 if (aodfiles.Contains(obj->GetName())) continue;
3638 if (outputfiles.Contains(obj->GetName())) continue;
3639 if (sextra.Contains(obj->GetName())) continue;
3640 if (!sextra.IsNull()) sextra += ",";
3641 sextra += obj->GetName();
3642 }
3643 delete fextra;
3644 if (stype.Contains("ext")) {
3645 if (!files.IsNull()) files += ",";
3646 files += sextra;
3647 }
3648 }
3649 if (stype == "ext") return files.Data();
3650 TString termfiles;
3651 if (!fTerminateFiles.IsNull()) {
3652 fTerminateFiles.Strip();
3653 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 3654 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 3655 TIter nextx(fextra);
3656 TObject *obj;
3657 while ((obj=nextx())) {
3658 if (aodfiles.Contains(obj->GetName())) continue;
3659 if (outputfiles.Contains(obj->GetName())) continue;
3660 if (termfiles.Contains(obj->GetName())) continue;
3661 if (sextra.Contains(obj->GetName())) continue;
3662 if (!termfiles.IsNull()) termfiles += ",";
3663 termfiles += obj->GetName();
3664 }
3665 delete fextra;
3666 }
3667 if (stype.Contains("ter")) {
3668 if (!files.IsNull() && !termfiles.IsNull()) {
3669 files += ",";
3670 files += termfiles;
3671 }
3672 }
3673 return files.Data();
3674}
3675
d2a409b2 3676//______________________________________________________________________________
a03be957 3677Bool_t AliAnalysisAlien::Submit()
d2a409b2 3678{
3679// Submit all master jobs.
3680 Int_t nmasterjobs = fInputFiles->GetEntries();
3681 Long_t tshoot = gSystem->Now();
a03be957 3682 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 3683 while (fNsubmitted < nmasterjobs) {
3684 Long_t now = gSystem->Now();
3685 if ((now-tshoot)>30000) {
3686 tshoot = now;
a03be957 3687 if (!SubmitNext()) return kFALSE;
d2a409b2 3688 }
3689 }
a03be957 3690 return kTRUE;
d2a409b2 3691}
3692
0f389141 3693//______________________________________________________________________________
a03be957 3694Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 3695{
3696// Submit all merging jobs.
dc030a24 3697 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3698 gGrid->Cd(fGridOutputDir);
3699 TString mergeJDLName = fExecutable;
3700 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 3701 if (!fInputFiles) {
3702 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3703 return kFALSE;
3704 }
0f389141 3705 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 3706 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 3707 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3708 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 3709 if (fOutputToRunNo) {
3710 // The output directory is the run number
3711 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3712 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3713 } else {
17b4e9a4 3714 if (!fRunNumbers.Length() && !fRunRange[0]) {
3715 // The output directory is the grid outdir
3716 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3717 runOutDir = fGridOutputDir;
3718 } else {
3719 // The output directory is the master number in 3 digits format
3720 printf("### Submitting merging job for master <%03d>\n", i);
3721 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3722 }
7c2cd90a 3723 }
3724 // Check now the number of merging stages.
2b222f7c 3725 TObjArray *list = fOutputFiles.Tokenize(",");
3726 TIter next(list);
3727 TObjString *str;
3728 TString outputFile;
3729 while((str=(TObjString*)next())) {
3730 outputFile = str->GetString();
3731 Int_t index = outputFile.Index("@");
3732 if (index > 0) outputFile.Remove(index);
37d21c01 3733 if (!fMergeExcludes.Contains(outputFile) &&
3734 !fRegisterExcludes.Contains(outputFile)) break;
2b222f7c 3735 }
3736 delete list;
b3e07543 3737 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 3738 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 3739 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 3740 }
a03be957 3741 if (!ntosubmit) return kTRUE;
d88e4159 3742 if (fDropToShell) {
3743 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3744 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3745 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3746 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3747 \n ################################################################################################################");
3748 gSystem->Exec("aliensh");
3749 } else {
3750 Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3751 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3752 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3753 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3754 \n ################################################################################################################");
3755 }
a03be957 3756 return kTRUE;
0f389141 3757}
3758
d2a409b2 3759//______________________________________________________________________________
a03be957 3760Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 3761{
08d5b699 3762// Submit next bunch of master jobs if the queue is free. The first master job is
3763// submitted right away, while the next will not be unless the previous was split.
3764// The plugin will not submit new master jobs if there are more that 500 jobs in
3765// waiting phase.
d2a409b2 3766 static Bool_t iscalled = kFALSE;
3767 static Int_t firstmaster = 0;
3768 static Int_t lastmaster = 0;
3769 static Int_t npermaster = 0;
a03be957 3770 if (iscalled) return kTRUE;
d2a409b2 3771 iscalled = kTRUE;
3772 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3773 Int_t ntosubmit = 0;
3774 TGridResult *res;
3775 TString jobID = "";
3bdcb562 3776 Int_t nmasterjobs = fInputFiles->GetEntries();
3777 if (!fNsubmitted) {
3778 ntosubmit = 1;
3779 if (!IsUseSubmitPolicy()) {
90d50a8c 3780 if (nmasterjobs>5)
3bdcb562 3781 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3782 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3783 ntosubmit = nmasterjobs;
3784 }
3785 } else {
d2a409b2 3786 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 3787 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 3788 // If last master not split, just return
a03be957 3789 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 3790 // No more than 100 waiting jobs
08d5b699 3791 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3792 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 3793 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 3794 if (!ntosubmit) ntosubmit = 1;
84fcd93f 3795 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 3796 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3797 }
d2a409b2 3798 for (Int_t i=0; i<ntosubmit; i++) {
3799 // Submit for a range of enumeration of runs.
a03be957 3800 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 3801 TString query;
cd11251e 3802 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3803 runOutDir.ReplaceAll(".xml", "");
3804 if (fOutputToRunNo)
3805 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3806 else
3807 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 3808 printf("********* %s\n",query.Data());
d2a409b2 3809 res = gGrid->Command(query);
3810 if (res) {
98ca124f 3811 TString cjobId1 = res->GetKey(0,"jobId");
3812 if (!cjobId1.Length()) {
d2a409b2 3813 iscalled = kFALSE;
a03be957 3814 gGrid->Stdout();
3815 gGrid->Stderr();
3816 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3817 return kFALSE;
d2a409b2 3818 } else {
3819 Info("StartAnalysis", "\n_______________________________________________________________________ \
3820 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3821 \n_______________________________________________________________________",
98ca124f 3822 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
0ad3ea1f 3823 if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3824 fGridJobIDs.Append(cjobId1);
3825 if (!fGridStages.IsNull()) fGridStages.Append(" ");
3826 fGridStages.Append("full");
d2a409b2 3827 jobID += cjobId1;
3828 jobID += " ";
98ca124f 3829 lastmaster = cjobId1.Atoi();
d2a409b2 3830 if (!firstmaster) firstmaster = lastmaster;
3831 fNsubmitted++;
3832 }
3833 delete res;
a03be957 3834 } else {
3835 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3836 return kFALSE;
d2a409b2 3837 }
3838 }
3839 iscalled = kFALSE;
a03be957 3840 return kTRUE;
d2a409b2 3841}
3842
c57f56b7 3843//______________________________________________________________________________
3844void AliAnalysisAlien::WriteAnalysisFile()
3845{
f10e8481 3846// Write current analysis manager into the file <analysisFile>
3847 TString analysisFile = fExecutable;
3848 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 3849 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3850 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3851 if (!mgr || !mgr->IsInitialized()) {
3852 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
3853 return;
3854 }
3855 // Check analysis type
3856 TObject *handler;
3857 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
3858 handler = (TObject*)mgr->GetInputEventHandler();
3859 if (handler) {
65eb22f8 3860 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3861 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3862 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3863 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3864 } else {
3865 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3866 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3867 }
c57f56b7 3868 }
3869 TDirectory *cdir = gDirectory;
f10e8481 3870 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3871 if (file) {
e16a394c 3872 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3873 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3874 // Unless merging makes no sense
3875 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3876 mgr->Write();
3877 delete file;
fe2d7fc2 3878 // Enable termination for local jobs
3879 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3880 }
3881 if (cdir) cdir->cd();
f10e8481 3882 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3883 }
3884 Bool_t copy = kTRUE;
d3b18c4c 3885 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3886 if (copy) {
3887 CdWork();
3888 TString workdir = gGrid->GetHomeDirectory();
3889 workdir += fGridWorkingDir;
d3b18c4c 3890 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3891 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
6c395669 3892 if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
3893 Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
c57f56b7 3894 }
3895}
3896
3897//______________________________________________________________________________
3898void AliAnalysisAlien::WriteAnalysisMacro()
3899{
3900// Write the analysis macro that will steer the analysis in grid mode.
3901 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3902 ofstream out;
3903 out.open(fAnalysisMacro.Data(), ios::out);
3904 if (!out.good()) {
3905 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3906 return;
3907 }
5b273635 3908 Bool_t hasSTEERBase = kFALSE;
3909 Bool_t hasESD = kFALSE;
3910 Bool_t hasAOD = kFALSE;
3911 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 3912 Bool_t hasOADB = kFALSE;
5b273635 3913 Bool_t hasANALYSISalice = kFALSE;
3914 Bool_t hasCORRFW = kFALSE;
c57f56b7 3915 TString func = fAnalysisMacro;
3916 TString type = "ESD";
3917 TString comment = "// Analysis using ";
c85cfc0f 3918 if (IsUseMCchain()) {
3919 type = "MC";
3920 comment += "MC";
3921 } else {
3922 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3923 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3924 type = "AOD";
3925 comment += "AOD";
3926 }
3927 }
0df6ccf2 3928 if (type!="AOD" && fFriendChainName!="") {
3929 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3930 return;
3931 }
c57f56b7 3932 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3933 else comment += " data";
3934 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3935 func.ReplaceAll(".C", "");
3936 out << "void " << func.Data() << "()" << endl;
3937 out << "{" << endl;
3938 out << comment.Data() << endl;
3939 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3940 out << " TStopwatch timer;" << endl;
3941 out << " timer.Start();" << endl << endl;
13f0c00b 3942 // Change temp directory to current one
fdee57f1 3943 if (!IsLocalTest()) {
3944 out << "// connect to AliEn and make the chain" << endl;
3945 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3946 }
13f0c00b 3947 out << "// Set temporary merging directory to current one" << endl;
3948 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3949 out << "// Set temporary compilation directory to current one" << endl;
3950 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3951 // Reset existing include path
3952 out << "// Reset existing include path and add current directory first in the search" << endl;
3953 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3954 if (!fExecutableCommand.Contains("aliroot")) {
3955 out << "// load base root libraries" << endl;
3956 out << " gSystem->Load(\"libTree\");" << endl;
3957 out << " gSystem->Load(\"libGeom\");" << endl;
3958 out << " gSystem->Load(\"libVMC\");" << endl;
3959 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3960 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3961 }
d5c6455a 3962 if (fAdditionalRootLibs.Length()) {
47a4137d 3963 // in principle libtree /lib geom libvmc etc. can go into this list, too
3964 out << "// Add aditional libraries" << endl;
3965 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3966 TIter next(list);
3967 TObjString *str;
3968 while((str=(TObjString*)next())) {
3969 if (str->GetString().Contains(".so"))
3970 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3971 }
47a4137d 3972 if (list) delete list;
d5c6455a 3973 }
57377eb5 3974 out << "// Load analysis framework libraries" << endl;
652561d5 3975 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3976 if (!fPackages) {
f790bc1b 3977 if (!fExecutableCommand.Contains("aliroot")) {
3978 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3979 out << " gSystem->Load(\"libESD\");" << endl;
3980 out << " gSystem->Load(\"libAOD\");" << endl;
3981 }
4e5c5506 3982 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 3983 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3984 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3985 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3986 } else {
4e5c5506 3987 TIter next(fPackages);
3988 TObject *obj;
57377eb5 3989 TString pkgname;
57377eb5 3990 while ((obj=next())) {
3991 pkgname = obj->GetName();
4478e6f1 3992 if (pkgname == "STEERBase" ||
3993 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3994 if (pkgname == "ESD" ||
3995 pkgname == "ESD.par") hasESD = kTRUE;
3996 if (pkgname == "AOD" ||
3997 pkgname == "AOD.par") hasAOD = kTRUE;
3998 if (pkgname == "ANALYSIS" ||
3999 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4000 if (pkgname == "OADB" ||
4001 pkgname == "OADB.par") hasOADB = kTRUE;
4478e6f1 4002 if (pkgname == "ANALYSISalice" ||
4003 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4004 if (pkgname == "CORRFW" ||
4005 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 4006 }
4007 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 4008 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4009 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 4010 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4011 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 4012 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4013 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 4014 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4015 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4016 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4017 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
57377eb5 4018 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4019 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 4020 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4021 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 4022 out << "// Compile other par packages" << endl;
4023 next.Reset();
fcc9bb6f 4024 while ((obj=next())) {
4025 pkgname = obj->GetName();
4478e6f1 4026 if (pkgname == "STEERBase" ||
4027 pkgname == "STEERBase.par" ||
4028 pkgname == "ESD" ||
4029 pkgname == "ESD.par" ||
4030 pkgname == "AOD" ||
4031 pkgname == "AOD.par" ||
4032 pkgname == "ANALYSIS" ||
4033 pkgname == "ANALYSIS.par" ||
7ea40ac8 4034 pkgname == "OADB" ||
4035 pkgname == "OADB.par" ||
4478e6f1 4036 pkgname == "ANALYSISalice" ||
4037 pkgname == "ANALYSISalice.par" ||
4038 pkgname == "CORRFW" ||
4039 pkgname == "CORRFW.par") continue;
5b273635 4040 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 4041 }
4e5c5506 4042 }
b3e07543 4043 out << "// include path" << endl;
4044 // Get the include path from the interpreter and remove entries pointing to AliRoot
4045 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4046 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4047 out << " TIter nextpath(listpaths);" << endl;
4048 out << " TObjString *pname;" << endl;
4049 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4050 out << " TString current = pname->GetName();" << endl;
4051 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4052 out << " gSystem->AddIncludePath(current);" << endl;
4053 out << " }" << endl;
4054 out << " if (listpaths) delete listpaths;" << endl;
4055 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4056 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4057 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 4058 if (fAdditionalLibs.Length()) {
4059 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4060 TString additionalLibs = fAdditionalLibs;
4061 additionalLibs.Strip();
4062 if (additionalLibs.Length() && fFriendLibs.Length())
4063 additionalLibs += " ";
4064 additionalLibs += fFriendLibs;
4065 TObjArray *list = additionalLibs.Tokenize(" ");
6da75e0b 4066 TIter next(list);
4067 TObjString *str;
4068 while((str=(TObjString*)next())) {
4069 if (str->GetString().Contains(".so"))
4070 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 4071 if (str->GetString().Contains(".par"))
4072 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 4073 }
4074 if (list) delete list;
4075 }
4076 out << endl;
c57f56b7 4077 out << "// analysis source to be compiled at runtime (if any)" << endl;
4078 if (fAnalysisSource.Length()) {
4079 TObjArray *list = fAnalysisSource.Tokenize(" ");
4080 TIter next(list);
4081 TObjString *str;
4082 while((str=(TObjString*)next())) {
4083 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4084 }
4085 if (list) delete list;
4086 }
4087 out << endl;
b3e07543 4088// out << " printf(\"Currently load libraries:\\n\");" << endl;
4089// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 4090 if (fFastReadOption) {
d3339be3 4091 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4092 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 4093 out << "// fast xrootd reading enabled" << endl;
4094 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4095 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4096 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4097 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4098 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4099 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
21ca8e59 4100 }
c57f56b7 4101 out << "// read the analysis manager from file" << endl;
f10e8481 4102 TString analysisFile = fExecutable;
4103 analysisFile.ReplaceAll(".sh", ".root");
105631b5 4104 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4105 << analysisFile << "\");" << endl;
4106 out << " if (!mgr) return;" << endl;
21ca8e59 4107 if (IsLocalTest()) {
4108 out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4109 out << " plugin->SetRunMode(\"test\");" << endl;
139fe6ba 4110 if (fFileForTestMode.IsNull())
4111 out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4112 else
4113 out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
922b1fc7 4114 out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4b5d9e21 4115 if (!fFriendChainName.IsNull())
4116 out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\");" << endl;
024dfa52 4117 if (IsUseMCchain())
4118 out << " plugin->SetUseMCchain();" << endl;
21ca8e59 4119 out << " mgr->SetGridHandler(plugin);" << endl;
627a5dbe 4120 if (AliAnalysisManager::GetAnalysisManager()) {
7a366e0b 4121 out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
627a5dbe 4122 out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4123 } else {
7a366e0b 4124 out << " mgr->SetDebugLevel(10);" << endl;
627a5dbe 4125 out << " mgr->SetNSysInfo(100);" << endl;
4126 }
21ca8e59 4127 }
c57f56b7 4128 out << " mgr->PrintStatus();" << endl;
52b6a92b 4129 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 4130 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 4131 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 4132 } else {
9a8288ff 4133 if (TestBit(AliAnalysisGrid::kTest))
4134 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4135 else
4136 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 4137 }
4138 }
21ca8e59 4139 if (!IsLocalTest()) {
242accb2 4140 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
21ca8e59 4141 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
4142 } else {
4143 out << " mgr->StartAnalysis(\"localfile\");" << endl;
242accb2 4144 }
f7498086 4145 out << " timer.Stop();" << endl;
4146 out << " timer.Print();" << endl;
c57f56b7 4147 out << "}" << endl << endl;
21ca8e59 4148 if (!IsLocalTest()) {
fcc9bb6f 4149 out <<"//________________________________________________________________________________" << endl;
c57f56b7 4150 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
4151 out << "{" << endl;
4152 out << "// Create a chain using url's from xml file" << endl;
242accb2 4153 out << " TString filename;" << endl;
4154 out << " Int_t run = 0;" << endl;
c85cfc0f 4155 if (IsUseMCchain()) {
4156 out << " TString treename = \"TE\";" << endl;
4157 } else {
4158 out << " TString treename = type;" << endl;
4159 out << " treename.ToLower();" << endl;
4160 out << " treename += \"Tree\";" << endl;
4161 }
e02fee64 4162 out << " printf(\"***************************************\\n\");" << endl;
4163 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4164 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 4165 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
4166 out << " if (!coll) {" << endl;
4167 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4168 out << " return NULL;" << endl;
4169 out << " }" << endl;
242accb2 4170 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 4171 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 4172 if(fFriendChainName!="") {
4173 out << " TChain *chainFriend = new TChain(treename);" << endl;
4174 }
c57f56b7 4175 out << " coll->Reset();" << endl;
0df6ccf2 4176 out << " while (coll->Next()) {" << endl;
242accb2 4177 out << " filename = coll->GetTURL("");" << endl;
4178 out << " if (mgr) {" << endl;
4179 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4180 out << " if (nrun && nrun != run) {" << endl;
4181 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4182 out << " mgr->SetRunFromPath(nrun);" << endl;
4183 out << " run = nrun;" << endl;
4184 out << " }" << endl;
4185 out << " }" << endl;
4186 out << " chain->Add(filename);" << endl;
0df6ccf2 4187 if(fFriendChainName!="") {
4188 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
4b5d9e21 4189 out << " if (fileFriend.Index(\"#\") > -1) fileFriend.Remove(fileFriend.Index(\"#\"));" << endl;
4190 out << " fileFriend = gSystem->DirName(fileFriend);" << endl;
4191 out << " fileFriend += \"/\";" << endl;
4192 out << " fileFriend += \"" << fFriendChainName << "\";";
4193 out << " TFile *file = TFile::Open(fileFriend);" << endl;
4194 out << " if (file) {" << endl;
4195 out << " file->Close();" << endl;
4196 out << " chainFriend->Add(fileFriend.Data());" << endl;
4197 out << " } else {" << endl;
4198 out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4199 out << " return 0;" << endl;
4200 out << " }" << endl;
0df6ccf2 4201 }
4202 out << " }" << endl;
c57f56b7 4203 out << " if (!chain->GetNtrees()) {" << endl;
4204 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4205 out << " return NULL;" << endl;
4206 out << " }" << endl;
0df6ccf2 4207 if(fFriendChainName!="") {
4208 out << " chain->AddFriend(chainFriend);" << endl;
4209 }
c57f56b7 4210 out << " return chain;" << endl;
fcc9bb6f 4211 out << "}" << endl << endl;
c57f56b7 4212 }
5b273635 4213 if (hasANALYSISalice) {
4214 out <<"//________________________________________________________________________________" << endl;
4215 out << "Bool_t SetupPar(const char *package) {" << endl;
4216 out << "// Compile the package and set it up." << endl;
4217 out << " TString pkgdir = package;" << endl;
4218 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4219 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4220 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4221 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4222 out << " // Check for BUILD.sh and execute" << endl;
4223 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4224 out << " printf(\"*******************************\\n\");" << endl;
4225 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4226 out << " printf(\"*******************************\\n\");" << endl;
4227 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4228 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4229 out << " gSystem->ChangeDirectory(cdir);" << endl;
4230 out << " return kFALSE;" << endl;
4231 out << " }" << endl;
4232 out << " } else {" << endl;
4233 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4234 out << " gSystem->ChangeDirectory(cdir);" << endl;
4235 out << " return kFALSE;" << endl;
4236 out << " }" << endl;
4237 out << " // Check for SETUP.C and execute" << endl;
4238 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4239 out << " printf(\"*******************************\\n\");" << endl;
4240 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4241 out << " printf(\"*******************************\\n\");" << endl;
4242 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4243 out << " } else {" << endl;
4244 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4245 out << " gSystem->ChangeDirectory(cdir);" << endl;
4246 out << " return kFALSE;" << endl;
4247 out << " }" << endl;
4248 out << " // Restore original workdir" << endl;
4249 out << " gSystem->ChangeDirectory(cdir);" << endl;
4250 out << " return kTRUE;" << endl;
4251 out << "}" << endl;
4252 }
c57f56b7 4253 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4254 }
4255 Bool_t copy = kTRUE;
d3b18c4c 4256 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4257 if (copy) {
4258 CdWork();
4259 TString workdir = gGrid->GetHomeDirectory();
4260 workdir += fGridWorkingDir;
4261 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
c57f56b7 4262 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
6c395669 4263// TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4264 if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4265 Form("alien://%s/%s", workdir.Data(),
4266 fAnalysisMacro.Data()))) Fatal("","Terminating");
c57f56b7 4267 }
4268}
4269
0f389141 4270//______________________________________________________________________________
4271void AliAnalysisAlien::WriteMergingMacro()
4272{
4273// Write a macro to merge the outputs per master job.
4274 if (!fMergeViaJDL) return;
4275 if (!fOutputFiles.Length()) {
4276 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4277 return;
4278 }
ec5acdce 4279 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
0f389141 4280 TString mergingMacro = fExecutable;
4281 mergingMacro.ReplaceAll(".sh","_merge.C");
139fe6ba 4282 if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 4283 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4284 ofstream out;
4285 out.open(mergingMacro.Data(), ios::out);
4286 if (!out.good()) {
4287 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4288 return;
4289 }
5b273635 4290 Bool_t hasSTEERBase = kFALSE;
4291 Bool_t hasESD = kFALSE;
4292 Bool_t hasAOD = kFALSE;
4293 Bool_t hasANALYSIS = kFALSE;
7ea40ac8 4294 Bool_t hasOADB = kFALSE;
5b273635 4295 Bool_t hasANALYSISalice = kFALSE;
4296 Bool_t hasCORRFW = kFALSE;
0f389141 4297 TString func = mergingMacro;
4298 TString comment;
4299 func.ReplaceAll(".C", "");
f47d5cb4 4300 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 4301 out << "{" << endl;
4302 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4303 out << " TStopwatch timer;" << endl;
4304 out << " timer.Start();" << endl << endl;
b3e07543 4305 // Reset existing include path
4306 out << "// Reset existing include path and add current directory first in the search" << endl;
4307 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 4308 if (!fExecutableCommand.Contains("aliroot")) {
4309 out << "// load base root libraries" << endl;
4310 out << " gSystem->Load(\"libTree\");" << endl;
4311 out << " gSystem->Load(\"libGeom\");" << endl;
4312 out << " gSystem->Load(\"libVMC\");" << endl;
4313 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4314 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4315 }
0f389141 4316 if (fAdditionalRootLibs.Length()) {
4317 // in principle libtree /lib geom libvmc etc. can go into this list, too
4318 out << "// Add aditional libraries" << endl;
4319 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4320 TIter next(list);
4321 TObjString *str;
4322 while((str=(TObjString*)next())) {
4323 if (str->GetString().Contains(".so"))
4324 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4325 }
4326 if (list) delete list;
4327 }
0f389141 4328 out << "// Load analysis framework libraries" << endl;
4329 if (!fPackages) {
7c2cd90a 4330 if (!fExecutableCommand.Contains("aliroot")) {
4331 out << " gSystem->Load(\"libSTEERBase\");" << endl;
4332 out << " gSystem->Load(\"libESD\");" << endl;
4333 out << " gSystem->Load(\"libAOD\");" << endl;
4334 }
0f389141 4335 out << " gSystem->Load(\"libANALYSIS\");" << endl;
7ea40ac8 4336 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4337 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4338 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4339 } else {
4340 TIter next(fPackages);
4341 TObject *obj;
4342 TString pkgname;
5b273635 4343 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 4344 while ((obj=next())) {
4345 pkgname = obj->GetName();
4346 if (pkgname == "STEERBase" ||
4347 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4348 if (pkgname == "ESD" ||
4349 pkgname == "ESD.par") hasESD = kTRUE;
4350 if (pkgname == "AOD" ||
4351 pkgname == "AOD.par") hasAOD = kTRUE;
4352 if (pkgname == "ANALYSIS" ||
4353 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
7ea40ac8 4354 if (pkgname == "OADB" ||
4355 pkgname == "OADB.par") hasOADB = kTRUE;
0f389141 4356 if (pkgname == "ANALYSISalice" ||
4357 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4358 if (pkgname == "CORRFW" ||
4359 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4360 }
5b273635 4361 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 4362 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 4363 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 4364 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 4365 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 4366 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 4367 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 4368 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 4369 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 4370 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
7ea40ac8 4371 if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4372 else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
0f389141 4373 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 4374 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 4375 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 4376 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 4377 out << "// Compile other par packages" << endl;
4378 next.Reset();
4379 while ((obj=next())) {
4380 pkgname = obj->GetName();
4381 if (pkgname == "STEERBase" ||
4382 pkgname == "STEERBase.par" ||
4383 pkgname == "ESD" ||
4384 pkgname == "ESD.par" ||
4385 pkgname == "AOD" ||
4386 pkgname == "AOD.par" ||
4387 pkgname == "ANALYSIS" ||
4388 pkgname == "ANALYSIS.par" ||
7ea40ac8 4389 pkgname == "OADB" ||
4390 pkgname == "OADB.par" ||
0f389141 4391 pkgname == "ANALYSISalice" ||
4392 pkgname == "ANALYSISalice.par" ||
4393 pkgname == "CORRFW" ||
4394 pkgname == "CORRFW.par") continue;
5b273635 4395 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 4396 }
4397 }
b3e07543 4398 out << "// include path" << endl;
4399 // Get the include path from the interpreter and remove entries pointing to AliRoot
4400 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4401 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4402 out << " TIter nextpath(listpaths);" << endl;
4403 out << " TObjString *pname;" << endl;
4404 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4405 out << " TString current = pname->GetName();" << endl;
4406 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4407 out << " gSystem->AddIncludePath(current);" << endl;
4408 out << " }" << endl;
4409 out << " if (listpaths) delete listpaths;" << endl;
4410 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4411 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4412 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 4413 if (fAdditionalLibs.Length()) {
4414 out << "// Add aditional AliRoot libraries" << endl;
a41b2882 4415 TString additionalLibs = fAdditionalLibs;
4416 additionalLibs.Strip();
4417 if (additionalLibs.Length() && fFriendLibs.Length())
4418 additionalLibs += " ";
4419 additionalLibs += fFriendLibs;
4420 TObjArray *list = additionalLibs.Tokenize(" ");
0f389141 4421 TIter next(list);
4422 TObjString *str;
4423 while((str=(TObjString*)next())) {
4424 if (str->GetString().Contains(".so"))
4425 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
4426 }
4427 if (list) delete list;
4428 }
4429 out << endl;
4430 out << "// Analysis source to be compiled at runtime (if any)" << endl;
4431 if (fAnalysisSource.Length()) {
4432 TObjArray *list = fAnalysisSource.Tokenize(" ");
4433 TIter next(list);
4434 TObjString *str;
4435 while((str=(TObjString*)next())) {
4436 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4437 }
4438 if (list) delete list;
4439 }
149d288c 4440 out << endl;
4441
0f389141 4442 if (fFastReadOption) {
4443 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4444 out << "// fast xrootd reading enabled" << endl;
4445 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 4446 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4447 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 4448 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 4449 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 4450 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 4451 }
4452 // Change temp directory to current one
fdee57f1 4453 out << "// Connect to AliEn" << endl;
4454 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
e8b839ab 4455 out << "// Set temporary merging directory to current one" << endl;
4456 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 4457 out << "// Set temporary compilation directory to current one" << endl;
4458 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
7c2cd90a 4459 out << " TString outputDir = dir;" << endl;
f790bc1b 4460 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
37d21c01 4461 out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
58268c13 4462 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 4463 out << " TIter *iter = new TIter(list);" << endl;
4464 out << " TObjString *str;" << endl;
a2f5fc01 4465 out << " TString outputFile;" << endl;
0f389141 4466 out << " Bool_t merged = kTRUE;" << endl;
ec5acdce 4467 TString analysisFile = fExecutable;
4468 analysisFile.ReplaceAll(".sh", ".root");
4469 out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4470 << analysisFile << "\");" << endl;
4471 out << " if (!mgr) {" << endl;
4472 out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4473 out << " return;" << endl;
4474 out << " }" << endl;
d66aa2a9 4475 if (IsLocalTest()) {
4476 out << " printf(\"===================================\n\");" << endl;
4477 out << " printf(\"Testing merging...\\n\");" << endl;
4478 out << " printf(\"===================================\n\");" << endl;
4479 }
0f389141 4480 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 4481 out << " outputFile = str->GetString();" << endl;
7c2cd90a 4482 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 4483 out << " Int_t index = outputFile.Index(\"@\");" << endl;
4484 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 4485 out << " // Skip already merged outputs" << endl;
a2f5fc01 4486 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
d66aa2a9 4487 out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
0f389141 4488 out << " continue;" << endl;
4489 out << " }" << endl;
a2f5fc01 4490 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 4491 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 4492 out << " if (!merged) {" << endl;
a2f5fc01 4493 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 4494 out << " return;" << endl;
0f389141 4495 out << " }" << endl;
4496 out << " }" << endl;
d66aa2a9 4497 if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
ec5acdce 4498 out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4499 out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4500 }
7c2cd90a 4501 out << " // all outputs merged, validate" << endl;
4502 out << " ofstream out;" << endl;
4503 out << " out.open(\"outputs_valid\", ios::out);" << endl;
4504 out << " out.close();" << endl;
4505 out << " // read the analysis manager from file" << endl;
d66aa2a9 4506 if (IsLocalTest()) {
4507 out << " printf(\"===================================\n\");" << endl;
4508 out << " printf(\"Testing Terminate()...\\n\");" << endl;
4509 out << " printf(\"===================================\n\");" << endl;
4510 } else {
4511 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4512 }
5c4250fc 4513 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 4514 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 4515 out << " mgr->PrintStatus();" << endl;
ec5acdce 4516 if (mgr) {
4517 if (mgr->GetDebugLevel()>3) {
f866cba5 4518 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4519 } else {
9a8288ff 4520 if (TestBit(AliAnalysisGrid::kTest))
4521 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4522 else
4523 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 4524 }
4525 }
9a8288ff 4526 out << " TTree *tree = NULL;" << endl;
4527 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 4528 out << "}" << endl << endl;
5b273635 4529 if (hasANALYSISalice) {
4530 out <<"//________________________________________________________________________________" << endl;
4531 out << "Bool_t SetupPar(const char *package) {" << endl;
4532 out << "// Compile the package and set it up." << endl;
4533 out << " TString pkgdir = package;" << endl;
4534 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 4535 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 4536 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4537 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4538 out << " // Check for BUILD.sh and execute" << endl;
4539 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4540 out << " printf(\"*******************************\\n\");" << endl;
4541 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4542 out << " printf(\"*******************************\\n\");" << endl;
4543 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4544 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4545 out << " gSystem->ChangeDirectory(cdir);" << endl;
4546 out << " return kFALSE;" << endl;
4547 out << " }" << endl;
4548 out << " } else {" << endl;
4549 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4550 out << " gSystem->ChangeDirectory(cdir);" << endl;
4551 out << " return kFALSE;" << endl;
4552 out << " }" << endl;
4553 out << " // Check for SETUP.C and execute" << endl;
4554 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4555 out << " printf(\"*******************************\\n\");" << endl;
4556 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4557 out << " printf(\"*******************************\\n\");" << endl;
4558 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4559 out << " } else {" << endl;
4560 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4561 out << " gSystem->ChangeDirectory(cdir);" << endl;
4562 out << " return kFALSE;" << endl;
4563 out << " }" << endl;
4564 out << " // Restore original workdir" << endl;
4565 out << " gSystem->ChangeDirectory(cdir);" << endl;
4566 out << " return kTRUE;" << endl;
4567 out << "}" << endl;
4568 }
0f389141 4569 }
4570 Bool_t copy = kTRUE;
d3b18c4c 4571 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4572 if (copy) {
4573 CdWork();
4574 TString workdir = gGrid->GetHomeDirectory();
4575 workdir += fGridWorkingDir;
4576 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4577 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
6c395669 4578// TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4579 if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4580 Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
0f389141 4581 }
4582}
4583
4584//______________________________________________________________________________
4585Bool_t AliAnalysisAlien::SetupPar(const char *package)
4586{
205b201f 4587// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 4588// Note that for loading the compiled library. The current directory should have precedence in
4589// LD_LIBRARY_PATH
4590 TString pkgdir = package;
4591 pkgdir.ReplaceAll(".par","");
b50a41c3 4592 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 4593 TString cdir = gSystem->WorkingDirectory();
4594 gSystem->ChangeDirectory(pkgdir);
4595 // Check for BUILD.sh and execute
4596 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4597 printf("**************************************************\n");
4598 printf("*** Building PAR archive %s\n", package);
4599 printf("**************************************************\n");
4600 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4601 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4602 gSystem->ChangeDirectory(cdir);
4603 return kFALSE;
4604 }
4605 } else {
4606 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4607 gSystem->ChangeDirectory(cdir);
4608 return kFALSE;
4609 }
4610 // Check for SETUP.C and execute
4611 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4612 printf("**************************************************\n");
4613 printf("*** Setup PAR archive %s\n", package);
4614 printf("**************************************************\n");
4615 gROOT->Macro("PROOF-INF/SETUP.C");
4616 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4617 } else {
4618 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4619 gSystem->ChangeDirectory(cdir);
4620 return kFALSE;
4621 }
4622 // Restore original workdir
4623 gSystem->ChangeDirectory(cdir);
4624 return kTRUE;
4625}
4626
c57f56b7 4627//______________________________________________________________________________
4628void AliAnalysisAlien::WriteExecutable()
4629{
4630// Generate the alien executable script.
5aa94dc3 4631 // Patch executable with -x to catch error code
4632 if (fExecutableCommand.Contains("root") &&
4633 fExecutableCommand.Contains("-q") &&
4634 !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
c57f56b7 4635 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4636 ofstream out;
4637 out.open(fExecutable.Data(), ios::out);
4638 if (out.bad()) {
5513444a 4639 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 4640 return;
4641 }
4642 out << "#!/bin/bash" << endl;
2a4d5166 4643 // Make sure we can properly compile par files
4644 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 4645 out << "echo \"=========================================\"" << endl;
4646 out << "echo \"############## PATH : ##############\"" << endl;
4647 out << "echo $PATH" << endl;
4648 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4649 out << "echo $LD_LIBRARY_PATH" << endl;
4650 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4651 out << "echo $ROOTSYS" << endl;
4652 out << "echo \"############## which root : ##############\"" << endl;
4653 out << "which root" << endl;
4654 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4655 out << "echo $ALICE_ROOT" << endl;
4656 out << "echo \"############## which aliroot : ##############\"" << endl;
4657 out << "which aliroot" << endl;
9c5ddadc 4658 out << "echo \"############## system limits : ##############\"" << endl;
4659 out << "ulimit -a" << endl;
4660 out << "echo \"############## memory : ##############\"" << endl;
4661 out << "free -m" << endl;
c57f56b7 4662 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 4663 out << fExecutableCommand << " ";
370c9880 4664 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4665 out << "RET=$?" << endl;
4666 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4667 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4668 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4669 out << " let sig=\"$RET - 128\""<<endl;
4670 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4671 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4672 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4673 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4674 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4675 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4676 out << " fi"<<endl;
4677 out << " exit $RET"<< endl;
4678 out << "fi" << endl << endl ;
4679 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
9c5ddadc 4680 out << "echo \"############## memory after: ##############\"" << endl;
4681 out << "free -m" << endl;
c57f56b7 4682 }
4683 Bool_t copy = kTRUE;
d3b18c4c 4684 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4685 if (copy) {
4686 CdWork();
4687 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 4688 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4689 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 4690 workdir += fGridWorkingDir;
4691 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
4692 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4693 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
6c395669 4694// TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
4695 if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4696 executable.Data())) Fatal("","Terminating");
c57f56b7 4697 }
4698}
4699
0f389141 4700//______________________________________________________________________________
4701void AliAnalysisAlien::WriteMergeExecutable()
4702{
4703// Generate the alien executable script for the merging job.
4704 if (!fMergeViaJDL) return;
4705 TString mergeExec = fExecutable;
4706 mergeExec.ReplaceAll(".sh", "_merge.sh");
4707 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4708 ofstream out;
4709 out.open(mergeExec.Data(), ios::out);
4710 if (out.bad()) {
4711 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4712 return;
4713 }
4714 out << "#!/bin/bash" << endl;
2a4d5166 4715 // Make sure we can properly compile par files
4716 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 4717 out << "echo \"=========================================\"" << endl;
4718 out << "echo \"############## PATH : ##############\"" << endl;
4719 out << "echo $PATH" << endl;
4720 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4721 out << "echo $LD_LIBRARY_PATH" << endl;
4722 out << "echo \"############## ROOTSYS : ##############\"" << endl;
4723 out << "echo $ROOTSYS" << endl;
4724 out << "echo \"############## which root : ##############\"" << endl;
4725 out << "which root" << endl;
4726 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4727 out << "echo $ALICE_ROOT" << endl;
4728 out << "echo \"############## which aliroot : ##############\"" << endl;
4729 out << "which aliroot" << endl;
4730 out << "echo \"############## system limits : ##############\"" << endl;
4731 out << "ulimit -a" << endl;
4732 out << "echo \"############## memory : ##############\"" << endl;
4733 out << "free -m" << endl;
4734 out << "echo \"=========================================\"" << endl << endl;
0f389141 4735 TString mergeMacro = fExecutable;
4736 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 4737 if (IsOneStageMerging())
4738 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4739 else
f47d5cb4 4740 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 4741 out << fExecutableCommand << " " << "$ARG" << endl;
370c9880 4742 out << "RET=$?" << endl;
4743 out << "if [ \"$RET\" != \"0\" ];then" << endl;
4744 out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4745 out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4746 out << " let sig=\"$RET - 128\""<<endl;
4747 out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4748 out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4749 out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4750 out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4751 out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4752 out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4753 out << " fi"<<endl;
4754 out << " exit $RET"<< endl;
4755 out << "fi" << endl << endl ;
0f389141 4756 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4757 out << "echo \"############## memory after: ##############\"" << endl;
4758 out << "free -m" << endl;
0f389141 4759 }
4760 Bool_t copy = kTRUE;
d3b18c4c 4761 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 4762 if (copy) {
4763 CdWork();
4764 TString workdir = gGrid->GetHomeDirectory();
4765 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 4766 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 4767 workdir += fGridWorkingDir;
4768 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
4769 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 4770 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
6c395669 4771// TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
4772 if (!copyLocal2Alien("WriteMergeExecutable",
4773 mergeExec.Data(), executable.Data())) Fatal("","Terminating");
0f389141 4774 }
4775}
4776
c57f56b7 4777//______________________________________________________________________________
5513444a 4778void AliAnalysisAlien::WriteProductionFile(const char *filename) const
4779{
4780// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 4781// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 4782// Next lines: full_path_to_dataset XXX (XXX is a string)
4783// To submit, one has to: submit jdl XXX for all lines
4784 ofstream out;
4785 out.open(filename, ios::out);
4786 if (out.bad()) {
4787 Error("WriteProductionFile", "Bad file name: %s", filename);
4788 return;
4789 }
d3b18c4c 4790 TString workdir;
4791 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
4792 workdir = gGrid->GetHomeDirectory();
5513444a 4793 workdir += fGridWorkingDir;
f5e8c702 4794 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 4795 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 4796 out << locjdl << " " << njobspermaster << endl;
5513444a 4797 Int_t nmasterjobs = fInputFiles->GetEntries();
4798 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 4799 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
4800 runOutDir.ReplaceAll(".xml", "");
4801 if (fOutputToRunNo)
4802 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
4803 else
4804 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 4805 }
d3b18c4c 4806 if (gGrid) {
4807 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
4808 if (FileExists(filename)) gGrid->Rm(filename);
6c395669 4809// TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
4810 if (!copyLocal2Alien("WriteProductionFile", filename,
4811 Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
d3b18c4c 4812 }
5513444a 4813}
4814
4815//______________________________________________________________________________
0f389141 4816void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 4817{
4818// Generate the alien validation script.
4819 // Generate the validation script
4820 TObjString *os;
d3b18c4c 4821 if (fValidationScript.IsNull()) {
4822 fValidationScript = fExecutable;
4823 fValidationScript.ReplaceAll(".sh", "_validation.sh");
4824 }
4825 TString validationScript = fValidationScript;
4826 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 4827 if (!Connect()) {
4828 Error("WriteValidationScript", "Alien connection required");
4829 return;
4830 }
d3b18c4c 4831 if (!fTerminateFiles.IsNull()) {
4832 fTerminateFiles.Strip();
4833 fTerminateFiles.ReplaceAll(" ",",");
4834 }
a2f5fc01 4835 TString outStream = "";
4836 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 4837 if (!TestBit(AliAnalysisGrid::kSubmit)) {
4838 ofstream out;
0d5d317c 4839 out.open(validationScript, ios::out);
c57f56b7 4840 out << "#!/bin/bash" << endl;
4841 out << "##################################################" << endl;
4842 out << "validateout=`dirname $0`" << endl;
4843 out << "validatetime=`date`" << endl;
4844 out << "validated=\"0\";" << endl;
4845 out << "error=0" << endl;
4846 out << "if [ -z $validateout ]" << endl;
4847 out << "then" << endl;
4848 out << " validateout=\".\"" << endl;
4849 out << "fi" << endl << endl;
4850 out << "cd $validateout;" << endl;
4851 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 4852 out << "echo \"*******************************************************\"" << outStream << endl;
4853 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 4854 out << "" << endl;
a2f5fc01 4855 out << "echo \"* Time: $validatetime \"" << outStream << endl;
4856 out << "echo \"* Dir: $validateout\"" << outStream << endl;
4857 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
4858 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4859 out << "ls -la ./" << outStream << endl;
4860 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 4861 out << "##################################################" << endl;
ebec370a 4862 out << "" << endl;
4863
4864 out << "if [ ! -f stderr ] ; then" << endl;
4865 out << " error=1" << endl;
a2f5fc01 4866 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
4867 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4868 out << "fi" << endl;
4869
b34c9f51 4870 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
4871 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
4872 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
4873 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
4874 out << "" << endl;
4875
ebec370a 4876 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
4877 out << " error=1" << endl;
a2f5fc01 4878 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
4879 out << " echo \"$parArch\" " << outStream << endl;
4880 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4881 out << "fi" << endl;
4882
4883 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
4884 out << " error=1" << endl;
a2f5fc01 4885 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
4886 out << " echo \"$segViol\" " << outStream << endl;
4887 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4888 out << "fi" << endl;
4889
4890 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
4891 out << " error=1" << endl;
a2f5fc01 4892 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
4893 out << " echo \"$segFault\" " << outStream << endl;
4894 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 4895 out << "fi" << endl;
4896
b34c9f51 4897 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
4898 out << " error=1" << endl;
a2f5fc01 4899 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
4900 out << " echo \"$glibcErr\" " << outStream << endl;
4901 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 4902 out << "fi" << endl;
4903
ebec370a 4904 // Part dedicated to the specific analyses running into the train
4905
d3b18c4c 4906 TString outputFiles = fOutputFiles;
4907 if (merge && !fTerminateFiles.IsNull()) {
4908 outputFiles += ",";
4909 outputFiles += fTerminateFiles;
4910 }
4911 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 4912 TIter next1(arr);
a2f5fc01 4913 TString outputFile;
bbd88db8 4914 while (!merge && (os=(TObjString*)next1())) {
4915 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4916 outputFile = os->GetString();
4917 Int_t index = outputFile.Index("@");
4918 if (index > 0) outputFile.Remove(index);
bbd88db8 4919 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4920 if (outputFile.Contains("*")) continue;
4921 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4922 out << " error=1" << endl;
7c2cd90a 4923 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4924 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4925 out << "fi" << endl;
4926 }
4927 delete arr;
7c2cd90a 4928 out << "if ! [ -f outputs_valid ] ; then" << endl;
4929 out << " error=1" << endl;
4930 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4931 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4932 out << "fi" << endl;
923e2ca5 4933
c57f56b7 4934 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4935 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4936 if (!IsKeepLogs()) {
4937 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4938 outStream = "";
149d288c 4939 out << " rm -f std*" << endl;
4940 }
c57f56b7 4941 out << "fi" << endl;
4942
a2f5fc01 4943 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4944 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4945 out << "cd -" << endl;
4946 out << "exit $error" << endl;
4947 }
4948 Bool_t copy = kTRUE;
d3b18c4c 4949 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4950 if (copy) {
4951 CdWork();
4952 TString workdir = gGrid->GetHomeDirectory();
4953 workdir += fGridWorkingDir;
d3b18c4c 4954 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4955 if (FileExists(validationScript)) gGrid->Rm(validationScript);
6c395669 4956// TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4957 if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
4958 Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
c57f56b7 4959 }
4960}