]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
added setter for harmonics
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
b3e07543 61 fMaxMergeStages(0),
d2a409b2 62 fNsubmitted(0),
a3e84053 63 fProductionMode(0),
cd11251e 64 fOutputToRunNo(0),
0f389141 65 fMergeViaJDL(0),
66 fFastReadOption(0),
e1c22e21 67 fOverwriteMode(1),
149d288c 68 fNreplicas(2),
3bdcb562 69 fNproofWorkers(0),
27734f0e 70 fNproofWorkersPerSlave(0),
3bdcb562 71 fProofReset(0),
c57f56b7 72 fRunNumbers(),
73 fExecutable(),
0a1c1f7f 74 fExecutableCommand(),
c57f56b7 75 fArguments(),
631c0b05 76 fExecutableArgs(),
c57f56b7 77 fAnalysisMacro(),
78 fAnalysisSource(),
d3b18c4c 79 fValidationScript(),
d5c6455a 80 fAdditionalRootLibs(),
c57f56b7 81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
648174cf 86 fExternalPackages(),
c57f56b7 87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
e7c71df0 95 fDatasetName(),
c57f56b7 96 fJDLName(),
d3b18c4c 97 fTerminateFiles(),
bb885a9e 98 fMergeExcludes(),
f965131e 99 fIncludePath(),
bb885a9e 100 fCloseSE(),
0df6ccf2 101 fFriendChainName(),
c6cb3634 102 fJobTag(),
648174cf 103 fOutputSingle(),
5fce53f4 104 fRunPrefix(),
3bdcb562 105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
f47d5cb4 110 fMergeDirName(),
4e5c5506 111 fInputFiles(0),
112 fPackages(0)
c57f56b7 113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
0f389141 122 fMergingJDL(NULL),
c57f56b7 123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
bb885a9e 128 fNtestFiles(0),
319593fb 129 fNrunsPerMaster(0),
16a4353c 130 fMaxMergeFiles(0),
b3e07543 131 fMaxMergeStages(0),
d2a409b2 132 fNsubmitted(0),
a3e84053 133 fProductionMode(0),
cd11251e 134 fOutputToRunNo(0),
0f389141 135 fMergeViaJDL(0),
136 fFastReadOption(0),
e1c22e21 137 fOverwriteMode(1),
149d288c 138 fNreplicas(2),
3bdcb562 139 fNproofWorkers(0),
27734f0e 140 fNproofWorkersPerSlave(0),
3bdcb562 141 fProofReset(0),
c57f56b7 142 fRunNumbers(),
143 fExecutable(),
0a1c1f7f 144 fExecutableCommand(),
c57f56b7 145 fArguments(),
631c0b05 146 fExecutableArgs(),
c57f56b7 147 fAnalysisMacro(),
148 fAnalysisSource(),
d3b18c4c 149 fValidationScript(),
d5c6455a 150 fAdditionalRootLibs(),
c57f56b7 151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
648174cf 156 fExternalPackages(),
c57f56b7 157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
e7c71df0 165 fDatasetName(),
c57f56b7 166 fJDLName(),
d3b18c4c 167 fTerminateFiles(),
bb885a9e 168 fMergeExcludes(),
f965131e 169 fIncludePath(),
bb885a9e 170 fCloseSE(),
0df6ccf2 171 fFriendChainName(),
c6cb3634 172 fJobTag(),
648174cf 173 fOutputSingle(),
5fce53f4 174 fRunPrefix(),
3bdcb562 175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
f47d5cb4 180 fMergeDirName(),
4e5c5506 181 fInputFiles(0),
182 fPackages(0)
c57f56b7 183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
0f389141 192 fMergingJDL(NULL),
c57f56b7 193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 198 fNtestFiles(other.fNtestFiles),
319593fb 199 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 200 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 201 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 202 fNsubmitted(other.fNsubmitted),
a3e84053 203 fProductionMode(other.fProductionMode),
cd11251e 204 fOutputToRunNo(other.fOutputToRunNo),
0f389141 205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
149d288c 208 fNreplicas(other.fNreplicas),
3bdcb562 209 fNproofWorkers(other.fNproofWorkers),
27734f0e 210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 211 fProofReset(other.fProofReset),
c57f56b7 212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
0a1c1f7f 214 fExecutableCommand(other.fExecutableCommand),
c57f56b7 215 fArguments(other.fArguments),
631c0b05 216 fExecutableArgs(other.fExecutableArgs),
c57f56b7 217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 219 fValidationScript(other.fValidationScript),
d5c6455a 220 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
648174cf 226 fExternalPackages(other.fExternalPackages),
c57f56b7 227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
e7c71df0 235 fDatasetName(other.fDatasetName),
c57f56b7 236 fJDLName(other.fJDLName),
d3b18c4c 237 fTerminateFiles(other.fTerminateFiles),
bb885a9e 238 fMergeExcludes(other.fMergeExcludes),
f965131e 239 fIncludePath(other.fIncludePath),
bb885a9e 240 fCloseSE(other.fCloseSE),
0df6ccf2 241 fFriendChainName(other.fFriendChainName),
c6cb3634 242 fJobTag(other.fJobTag),
648174cf 243 fOutputSingle(other.fOutputSingle),
5fce53f4 244 fRunPrefix(other.fRunPrefix),
3bdcb562 245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
f47d5cb4 250 fMergeDirName(other.fMergeDirName),
4e5c5506 251 fInputFiles(0),
252 fPackages(0)
c57f56b7 253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
c57f56b7 259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
4e5c5506 266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
c57f56b7 273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
0f389141 280 if (fMergingJDL) delete fMergingJDL;
c57f56b7 281 if (fInputFiles) delete fInputFiles;
4e5c5506 282 if (fPackages) delete fPackages;
c57f56b7 283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 298 fNtestFiles = other.fNtestFiles;
a3e84053 299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 301 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
cd11251e 304 fOutputToRunNo = other.fOutputToRunNo;
0f389141 305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
149d288c 308 fNreplicas = other.fNreplicas;
3bdcb562 309 fNproofWorkers = other.fNproofWorkers;
27734f0e 310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 311 fProofReset = other.fProofReset;
c57f56b7 312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
0a1c1f7f 314 fExecutableCommand = other.fExecutableCommand;
c57f56b7 315 fArguments = other.fArguments;
631c0b05 316 fExecutableArgs = other.fExecutableArgs;
c57f56b7 317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 319 fValidationScript = other.fValidationScript;
d5c6455a 320 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
648174cf 326 fExternalPackages = other.fExternalPackages;
c57f56b7 327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
e7c71df0 335 fDatasetName = other.fDatasetName;
c57f56b7 336 fJDLName = other.fJDLName;
d3b18c4c 337 fTerminateFiles = other.fTerminateFiles;
bb885a9e 338 fMergeExcludes = other.fMergeExcludes;
f965131e 339 fIncludePath = other.fIncludePath;
bb885a9e 340 fCloseSE = other.fCloseSE;
0df6ccf2 341 fFriendChainName = other.fFriendChainName;
c6cb3634 342 fJobTag = other.fJobTag;
648174cf 343 fOutputSingle = other.fOutputSingle;
5fce53f4 344 fRunPrefix = other.fRunPrefix;
3bdcb562 345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
f47d5cb4 350 fMergeDirName = other.fMergeDirName;
c57f56b7 351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
4e5c5506 358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
c57f56b7 365 }
366 return *this;
367}
368
74013535 369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
f965131e 377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 391 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 392}
393
ee75cfc3 394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunNumber(const char* run)
396{
397// Add a run number to the list of runs to be processed.
398 if (fRunNumbers.Length()) fRunNumbers += " ";
399 fRunNumbers += run;
400}
401
c57f56b7 402//______________________________________________________________________________
403void AliAnalysisAlien::AddDataFile(const char *lfn)
404{
405// Adds a data file to the input to be analysed. The file should be a valid LFN
406// or point to an existing file in the alien workdir.
407 if (!fInputFiles) fInputFiles = new TObjArray();
408 fInputFiles->Add(new TObjString(lfn));
409}
648174cf 410
411//______________________________________________________________________________
412void AliAnalysisAlien::AddExternalPackage(const char *package)
413{
414// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
415 if (fExternalPackages) fExternalPackages += " ";
416 fExternalPackages += package;
417}
418
c57f56b7 419//______________________________________________________________________________
420Bool_t AliAnalysisAlien::Connect()
421{
422// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
423 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 424 if (fProductionMode) return kTRUE;
c57f56b7 425 if (!gGrid) {
426 Info("Connect", "Trying to connect to AliEn ...");
427 TGrid::Connect("alien://");
428 }
429 if (!gGrid || !gGrid->IsConnected()) {
430 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
431 return kFALSE;
432 }
433 fUser = gGrid->GetUser();
434 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
435 return kTRUE;
436}
437
438//______________________________________________________________________________
439void AliAnalysisAlien::CdWork()
440{
441// Check validity of alien workspace. Create directory if possible.
442 if (!Connect()) {
443 Error("CdWork", "Alien connection required");
444 return;
445 }
446 TString homedir = gGrid->GetHomeDirectory();
447 TString workdir = homedir + fGridWorkingDir;
923e2ca5 448 if (DirectoryExists(workdir)) {
449 gGrid->Cd(workdir);
450 return;
451 }
452 // Work directory not existing - create it
453 gGrid->Cd(homedir);
b93f8109 454 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 455 gGrid->Cd(fGridWorkingDir);
d3b18c4c 456 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 457 } else {
d3b18c4c 458 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 459 workdir.Data(), homedir.Data());
460 fGridWorkingDir = "";
461 }
c57f56b7 462}
463
348be253 464//______________________________________________________________________________
465Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
466{
467// Check if file copying is possible.
d3b18c4c 468 if (fProductionMode) return kTRUE;
348be253 469 if (!Connect()) {
470 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
471 return kFALSE;
472 }
d3339be3 473 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
474 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 475 // Check if alien_CLOSE_SE is defined
476 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
477 if (!closeSE.IsNull()) {
478 Info("CheckFileCopy", "Your current close storage is pointing to: \
479 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
480 } else {
481 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
482 }
483 // Check if grid directory exists.
484 if (!DirectoryExists(alienpath)) {
485 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
486 return kFALSE;
487 }
d3339be3 488 TFile f("plugin_test_copy", "RECREATE");
348be253 489 // User may not have write permissions to current directory
490 if (f.IsZombie()) {
491 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
492 gSystem->WorkingDirectory());
493 return kFALSE;
494 }
495 f.Close();
d3339be3 496 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
497 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
498 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 499 \n# 1. Make sure you have write permissions there. If this is the case: \
500 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
501 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
502 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
503 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
504 gSystem->Unlink(f.GetName());
505 return kFALSE;
506 }
507 gSystem->Unlink(f.GetName());
d3339be3 508 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
509 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 510 return kTRUE;
511}
512
c57f56b7 513//______________________________________________________________________________
514Bool_t AliAnalysisAlien::CheckInputData()
515{
516// Check validity of input data. If necessary, create xml files.
d3b18c4c 517 if (fProductionMode) return kTRUE;
a8739e8a 518 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 519 if (!fGridDataDir.Length()) {
520 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
521 return kFALSE;
522 }
9f020952 523 if (fMergeViaJDL) {
524 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
525 return kFALSE;
526 }
d2a409b2 527 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 528 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
529 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 530 return kTRUE;
c57f56b7 531 }
532 // Process declared files
a2f5fc01 533 Bool_t isCollection = kFALSE;
534 Bool_t isXml = kFALSE;
535 Bool_t useTags = kFALSE;
c57f56b7 536 Bool_t checked = kFALSE;
d3b18c4c 537 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 538 TString file;
539 TString workdir = gGrid->GetHomeDirectory();
540 workdir += fGridWorkingDir;
541 if (fInputFiles) {
542 TObjString *objstr;
543 TIter next(fInputFiles);
544 while ((objstr=(TObjString*)next())) {
545 file = workdir;
546 file += "/";
547 file += objstr->GetString();
548 // Store full lfn path
549 if (FileExists(file)) objstr->SetString(file);
550 else {
551 file = objstr->GetName();
552 if (!FileExists(objstr->GetName())) {
553 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
554 objstr->GetName(), workdir.Data());
555 return kFALSE;
556 }
557 }
558 Bool_t iscoll, isxml, usetags;
559 CheckDataType(file, iscoll, isxml, usetags);
560 if (!checked) {
561 checked = kTRUE;
a2f5fc01 562 isCollection = iscoll;
563 isXml = isxml;
564 useTags = usetags;
565 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 566 } else {
a2f5fc01 567 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 568 Error("CheckInputData", "Some conflict was found in the types of inputs");
569 return kFALSE;
570 }
571 }
572 }
573 }
574 // Process requested run numbers
a8739e8a 575 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 576 // Check validity of alien data directory
577 if (!fGridDataDir.Length()) {
578 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
579 return kFALSE;
580 }
923e2ca5 581 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 582 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
583 return kFALSE;
584 }
a2f5fc01 585 if (isCollection) {
c57f56b7 586 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
587 return kFALSE;
588 }
589
a2f5fc01 590 if (checked && !isXml) {
c57f56b7 591 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
592 return kFALSE;
593 }
594 // Check validity of run number(s)
595 TObjArray *arr;
596 TObjString *os;
74013535 597 TString format;
319593fb 598 Int_t nruns = 0;
904f9f5f 599 TString schunk, schunk2;
c57f56b7 600 TString path;
601 if (!checked) {
602 checked = kTRUE;
a2f5fc01 603 useTags = fDataPattern.Contains("tag");
604 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 605 }
a2f5fc01 606 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 607 Error("CheckInputData", "Cannot mix input files using/not using tags");
608 return kFALSE;
609 }
610 if (fRunNumbers.Length()) {
a8739e8a 611 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 612 arr = fRunNumbers.Tokenize(" ");
613 TIter next(arr);
614 while ((os=(TObjString*)next())) {
615 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 616 if (!DirectoryExists(path)) {
617 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 618 continue;
c57f56b7 619 }
620 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
621 TString msg = "\n##### file: ";
622 msg += path;
623 msg += " type: xml_collection;";
a2f5fc01 624 if (useTags) msg += " using_tags: Yes";
c57f56b7 625 else msg += " using_tags: No";
23329835 626 Info("CheckDataType", "%s", msg.Data());
319593fb 627 if (fNrunsPerMaster<2) {
d2a409b2 628 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 629 } else {
630 nruns++;
631 if (((nruns-1)%fNrunsPerMaster) == 0) {
632 schunk = os->GetString();
633 }
634 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
635 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 636 AddDataFile(schunk);
319593fb 637 }
c57f56b7 638 }
639 delete arr;
a8739e8a 640 } else {
641 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
642 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 643 format = Form("%%s/%s ", fRunPrefix.Data());
644 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 645 if (!DirectoryExists(path)) {
a8739e8a 646 continue;
647 }
74013535 648 format = Form("%%s/%s.xml", fRunPrefix.Data());
649 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 650 TString msg = "\n##### file: ";
651 msg += path;
652 msg += " type: xml_collection;";
a2f5fc01 653 if (useTags) msg += " using_tags: Yes";
a8739e8a 654 else msg += " using_tags: No";
23329835 655 Info("CheckDataType", "%s", msg.Data());
319593fb 656 if (fNrunsPerMaster<2) {
74013535 657 format = Form("%s.xml", fRunPrefix.Data());
658 AddDataFile(Form(format.Data(),irun));
319593fb 659 } else {
660 nruns++;
661 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 662 schunk = Form(fRunPrefix.Data(),irun);
319593fb 663 }
74013535 664 format = Form("_%s.xml", fRunPrefix.Data());
665 schunk2 = Form(format.Data(), irun);
319593fb 666 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 667 schunk += schunk2;
d2a409b2 668 AddDataFile(schunk);
319593fb 669 }
a8739e8a 670 }
904f9f5f 671 if (!fInputFiles) {
672 schunk += schunk2;
673 AddDataFile(schunk);
674 }
c57f56b7 675 }
676 return kTRUE;
677}
678
679//______________________________________________________________________________
680Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
681{
682// Create dataset for the grid data directory + run number.
d3b18c4c 683 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 684 if (!Connect()) {
685 Error("CreateDataset", "Cannot create dataset with no grid connection");
686 return kFALSE;
687 }
688
689 // Cd workspace
d3b18c4c 690 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 691 TString workdir = gGrid->GetHomeDirectory();
692 workdir += fGridWorkingDir;
693
694 // Compose the 'find' command arguments
74013535 695 TString format;
c57f56b7 696 TString command;
697 TString options = "-x collection ";
bb885a9e 698 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 699 TString conditions = "";
700
701 TString file;
702 TString path;
319593fb 703 Int_t nruns = 0;
904f9f5f 704 TString schunk, schunk2;
ab254fd1 705 TGridCollection *cbase=0, *cadd=0;
d2a409b2 706 if (!fRunNumbers.Length() && !fRunRange[0]) {
707 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
708 // Make a single data collection from data directory.
709 path = fGridDataDir;
923e2ca5 710 if (!DirectoryExists(path)) {
d2a409b2 711 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
712 return kFALSE;
713 }
923e2ca5 714// CdWork();
d2a409b2 715 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
716 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 717 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 718 command = "find ";
719 command += options;
720 command += path;
721 command += " ";
722 command += pattern;
723 command += conditions;
84fcd93f 724 printf("command: %s\n", command.Data());
d2a409b2 725 TGridResult *res = gGrid->Command(command);
726 if (res) delete res;
727 // Write standard output to file
728 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 729 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
730 Bool_t nullFile = kFALSE;
731 if (!hasGrep) {
d3339be3 732 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
733 } else {
a2f5fc01 734 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
735 if (nullFile) {
defd7a3a 736 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
737 return kFALSE;
738 }
d3339be3 739 }
0f389141 740 }
741 Bool_t fileExists = FileExists(file);
742 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 743 // Copy xml file to alien space
0f389141 744 if (fileExists) gGrid->Rm(file);
d2a409b2 745 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
746 if (!FileExists(file)) {
747 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
748 return kFALSE;
749 }
750 // Update list of files to be processed.
751 }
752 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
753 return kTRUE;
754 }
c57f56b7 755 // Several runs
a2f5fc01 756 Bool_t nullResult = kTRUE;
a8739e8a 757 if (fRunNumbers.Length()) {
758 TObjArray *arr = fRunNumbers.Tokenize(" ");
759 TObjString *os;
760 TIter next(arr);
761 while ((os=(TObjString*)next())) {
762 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 763 if (!DirectoryExists(path)) continue;
764// CdWork();
a8739e8a 765 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
766 else file = Form("%s.xml", os->GetString().Data());
319593fb 767 // If local collection file does not exist, create it via 'find' command.
d3339be3 768 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 769 command = "find ";
770 command += options;
771 command += path;
772 command += pattern;
773 command += conditions;
774 TGridResult *res = gGrid->Command(command);
775 if (res) delete res;
776 // Write standard output to file
777 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 778 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
779 Bool_t nullFile = kFALSE;
780 if (!hasGrep) {
d3339be3 781 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
782 } else {
a2f5fc01 783 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
784 if (nullFile) {
defd7a3a 785 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
786 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
787 continue;
788 }
d3339be3 789 }
a2f5fc01 790 nullResult = kFALSE;
d3339be3 791 }
a8739e8a 792 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 793 // Check if there is one run per master job.
794 if (fNrunsPerMaster<2) {
795 if (FileExists(file)) {
0f389141 796 if (fOverwriteMode) gGrid->Rm(file);
797 else {
798 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
799 continue;
800 }
319593fb 801 }
802 // Copy xml file to alien space
803 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
804 if (!FileExists(file)) {
805 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
806 delete arr;
807 return kFALSE;
808 }
809 } else {
810 nruns++;
811 if (((nruns-1)%fNrunsPerMaster) == 0) {
812 schunk = os->GetString();
813 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
814 } else {
815 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 816 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 817 cbase->Add(cadd);
818 delete cadd;
819 }
820 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
821 continue;
822 }
823 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 824 if (FileExists(schunk)) {
825 if (fOverwriteMode) gGrid->Rm(file);
826 else {
827 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
828 continue;
829 }
319593fb 830 }
84fcd93f 831 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 832 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 833 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 834 if (!FileExists(schunk)) {
835 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
836 delete arr;
837 return kFALSE;
838 }
d3339be3 839 }
a8739e8a 840 }
841 delete arr;
a2f5fc01 842 if (nullResult) {
d3339be3 843 Error("CreateDataset", "No valid dataset corresponding to the query!");
844 return kFALSE;
845 }
a8739e8a 846 } else {
847 // Process a full run range.
848 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 849 format = Form("%%s/%s ", fRunPrefix.Data());
850 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 851 if (!DirectoryExists(path)) continue;
852// CdWork();
74013535 853 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 854 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 855 else file = Form(format.Data(), irun);
0f389141 856 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
857 if (fOverwriteMode) gGrid->Rm(file);
858 else {
859 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
860 continue;
861 }
a8739e8a 862 }
319593fb 863 // If local collection file does not exist, create it via 'find' command.
d3339be3 864 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 865 command = "find ";
866 command += options;
867 command += path;
868 command += pattern;
869 command += conditions;
870 TGridResult *res = gGrid->Command(command);
871 if (res) delete res;
872 // Write standard output to file
873 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 874 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
875 Bool_t nullFile = kFALSE;
876 if (!hasGrep) {
d3339be3 877 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
878 } else {
a2f5fc01 879 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
880 if (nullFile) {
defd7a3a 881 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
882 continue;
883 }
d3339be3 884 }
a2f5fc01 885 nullResult = kFALSE;
319593fb 886 }
a8739e8a 887 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 888 // Check if there is one run per master job.
889 if (fNrunsPerMaster<2) {
890 if (FileExists(file)) {
0f389141 891 if (fOverwriteMode) gGrid->Rm(file);
892 else {
893 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
894 continue;
895 }
319593fb 896 }
897 // Copy xml file to alien space
898 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
899 if (!FileExists(file)) {
900 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
901 return kFALSE;
902 }
903 } else {
904 nruns++;
95e5b448 905 // Check if the collection for the chunk exist locally.
906 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 907 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
908 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
909 else continue;
910 }
84fcd93f 911 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 912 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 913 schunk = Form(fRunPrefix.Data(), irun);
319593fb 914 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
915 } else {
916 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
917 cbase->Add(cadd);
918 delete cadd;
919 }
74013535 920 format = Form("%%s_%s.xml", fRunPrefix.Data());
921 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 922 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 923 continue;
924 }
904f9f5f 925 schunk = schunk2;
319593fb 926 if (FileExists(schunk)) {
0f389141 927 if (fOverwriteMode) gGrid->Rm(schunk);
928 else {
929 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
930 continue;
931 }
319593fb 932 }
84fcd93f 933 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 934 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 935 if (FileExists(schunk)) {
0f389141 936 if (fOverwriteMode) gGrid->Rm(schunk);
937 else {
938 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
939 continue;
940 }
95e5b448 941 }
319593fb 942 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
943 if (!FileExists(schunk)) {
944 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
945 return kFALSE;
946 }
947 }
c57f56b7 948 }
a2f5fc01 949 if (nullResult) {
d3339be3 950 Error("CreateDataset", "No valid dataset corresponding to the query!");
951 return kFALSE;
952 }
a8739e8a 953 }
c57f56b7 954 return kTRUE;
955}
956
957//______________________________________________________________________________
958Bool_t AliAnalysisAlien::CreateJDL()
959{
960// Generate a JDL file according to current settings. The name of the file is
961// specified by fJDLName.
962 Bool_t error = kFALSE;
963 TObjArray *arr = 0;
964 Bool_t copy = kTRUE;
d3b18c4c 965 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 966 Bool_t generate = kTRUE;
967 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
968 if (!Connect()) {
969 Error("CreateJDL", "Alien connection required");
970 return kFALSE;
971 }
972 // Check validity of alien workspace
d3b18c4c 973 TString workdir;
974 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
975 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 976 workdir += fGridWorkingDir;
977 if (generate) {
978 TObjString *os;
979 if (!fInputFiles) {
980 Error("CreateJDL()", "Define some input files for your analysis.");
981 error = kTRUE;
982 }
983 // Compose list of input files
984 // Check if output files were defined
985 if (!fOutputFiles.Length()) {
986 Error("CreateJDL", "You must define at least one output file");
987 error = kTRUE;
988 }
989 // Check if an output directory was defined and valid
990 if (!fGridOutputDir.Length()) {
991 Error("CreateJDL", "You must define AliEn output directory");
992 error = kTRUE;
993 } else {
d3b18c4c 994 if (!fProductionMode) {
995 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
996 if (!DirectoryExists(fGridOutputDir)) {
997 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
998 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
999 } else {
1000 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1001 // error = kTRUE;
1002 }
67a3b254 1003 } else {
1004 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1005 }
1006 gGrid->Cd(workdir);
1007 }
c57f56b7 1008 }
1009 // Exit if any error up to now
1010 if (error) return kFALSE;
1011 // Set JDL fields
0f389141 1012 if (!fUser.IsNull()) {
1013 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1014 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1015 }
1016 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1017 TString mergeExec = fExecutable;
1018 mergeExec.ReplaceAll(".sh", "_merge.sh");
1019 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1020 mergeExec.ReplaceAll(".sh", ".C");
1021 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1022 if (!fArguments.IsNull())
1023 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1024 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1025 else {
1026 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1027 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1028 }
b3e07543 1029
1f0d1ca2 1030 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1031 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1032 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1033 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1034
0f389141 1035 if (fMaxInitFailed > 0) {
c57f56b7 1036 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1037 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1038 }
1039 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1040 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1041 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1042 }
1043 if (!IsOneStageMerging()) {
f47d5cb4 1044 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1045 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1046 }
1047 if (fSplitMode.Length()) {
c57f56b7 1048 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1049 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1050 }
1051 fMergingJDL->SetValue("Split", "\"se\"");
1052 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1053 if (!fAliROOTVersion.IsNull()) {
1054 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1055 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1056 }
1057 if (!fROOTVersion.IsNull()) {
c57f56b7 1058 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1059 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1060 }
1061 if (!fAPIVersion.IsNull()) {
c57f56b7 1062 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1063 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1064 }
648174cf 1065 if (!fExternalPackages.IsNull()) {
1066 arr = fExternalPackages.Tokenize(" ");
1067 TIter next(arr);
1068 while ((os=(TObjString*)next())) {
1069 TString pkgname = os->GetString();
1070 Int_t index = pkgname.Index("::");
1071 TString pkgversion = pkgname(index+2, pkgname.Length());
1072 pkgname.Remove(index);
1073 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1074 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1075 }
1076 delete arr;
1077 }
0f389141 1078 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1079 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1080 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1081 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1082 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1083 TString analysisFile = fExecutable;
1084 analysisFile.ReplaceAll(".sh", ".root");
1085 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1086 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1087 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1088 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1089 if (fAdditionalLibs.Length()) {
1090 arr = fAdditionalLibs.Tokenize(" ");
1091 TIter next(arr);
1092 while ((os=(TObjString*)next())) {
1093 if (os->GetString().Contains(".so")) continue;
1094 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1095 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1096 }
1097 delete arr;
1098 }
4e5c5506 1099 if (fPackages) {
1100 TIter next(fPackages);
1101 TObject *obj;
0f389141 1102 while ((obj=next())) {
4e5c5506 1103 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1104 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1105 }
4e5c5506 1106 }
c57f56b7 1107 if (fOutputArchive.Length()) {
1108 arr = fOutputArchive.Tokenize(" ");
1109 TIter next(arr);
0f389141 1110 Bool_t first = kTRUE;
1111 const char *comment = "Files to be archived";
1112 const char *comment1 = comment;
1113 while ((os=(TObjString*)next())) {
1114 if (!first) comment = NULL;
1115 if (!os->GetString().Contains("@") && fCloseSE.Length())
1116 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1117 else
1118 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1119 first = kFALSE;
1120 }
c57f56b7 1121 delete arr;
f790bc1b 1122 // Output archive for the merging jdl
1123 TString outputArchive;
1124 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1125 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1126 // Add normal output files, extra files + terminate files
1127 TString files = GetListOfFiles("outextter");
1128 // Do not register merge excludes
1129 if (!fMergeExcludes.IsNull()) {
1130 arr = fMergeExcludes.Tokenize(" ");
1131 TIter next1(arr);
1132 while ((os=(TObjString*)next1())) {
1133 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1134 files.ReplaceAll(os->GetString(),"");
1135 }
66d71516 1136 delete arr;
0f389141 1137 }
f790bc1b 1138 files.ReplaceAll(".root", "*.root");
b83f84f0 1139 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1140 } else {
95f84779 1141 TString files = fOutputArchive;
1142 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1143 outputArchive = files;
f790bc1b 1144 }
0f389141 1145 arr = outputArchive.Tokenize(" ");
705adb3e 1146 TIter next2(arr);
0f389141 1147 comment = comment1;
1148 first = kTRUE;
705adb3e 1149 while ((os=(TObjString*)next2())) {
0f389141 1150 if (!first) comment = NULL;
0cdf65a8 1151 TString currentfile = os->GetString();
0cdf65a8 1152 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1153 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1154 else
0cdf65a8 1155 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1156 first = kFALSE;
1157 }
1158 delete arr;
c57f56b7 1159 }
149d288c 1160 arr = fOutputFiles.Tokenize(",");
c57f56b7 1161 TIter next(arr);
0f389141 1162 Bool_t first = kTRUE;
f790bc1b 1163 const char *comment = "Files to be saved";
43da816a 1164 while ((os=(TObjString*)next())) {
e1eaf596 1165 // Ignore ouputs in jdl that are also in outputarchive
1166 TString sout = os->GetString();
f790bc1b 1167 sout.ReplaceAll("*", "");
1168 sout.ReplaceAll(".root", "");
e1eaf596 1169 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1170 if (fOutputArchive.Contains(sout)) continue;
0f389141 1171 if (!first) comment = NULL;
43da816a 1172 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1173 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1174 else
0f389141 1175 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1176 first = kFALSE;
f790bc1b 1177 if (fMergeExcludes.Contains(sout)) continue;
1178 if (!os->GetString().Contains("@") && fCloseSE.Length())
1179 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1180 else
1181 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1182 }
c57f56b7 1183 delete arr;
0f389141 1184 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1185 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1186 TString validationScript = fValidationScript;
0f389141 1187 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1188 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1189 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1190 if (fMasterResubmitThreshold) {
1191 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1192 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1193 }
d2a409b2 1194 // Write a jdl with 2 input parameters: collection name and output dir name.
1195 WriteJDL(copy);
c57f56b7 1196 }
1197 // Copy jdl to grid workspace
a8739e8a 1198 if (copy) {
b5e4aaa7 1199 // Check if an output directory was defined and valid
1200 if (!fGridOutputDir.Length()) {
1201 Error("CreateJDL", "You must define AliEn output directory");
1202 return kFALSE;
1203 } else {
1204 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1205 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1206 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1207 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1208 } else {
1209 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1210 return kFALSE;
1211 }
1212 }
1213 gGrid->Cd(workdir);
1214 }
648174cf 1215 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1216 TString mergeJDLName = fExecutable;
1217 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1218 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1219 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1220 if (fProductionMode) {
648174cf 1221 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1222 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1223 }
648174cf 1224 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1225 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1226 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1227 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1228 if (fMergeViaJDL) {
1229 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1230 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1231 }
648174cf 1232 }
c57f56b7 1233 if (fAdditionalLibs.Length()) {
e7c71df0 1234 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1235 TObjString *os;
1236 TIter next(arr);
1237 while ((os=(TObjString*)next())) {
c57f56b7 1238 if (os->GetString().Contains(".so")) continue;
4e5c5506 1239 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1240 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1241 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1242 }
1243 delete arr;
1244 }
4e5c5506 1245 if (fPackages) {
1246 TIter next(fPackages);
1247 TObject *obj;
1248 while ((obj=next())) {
fdbbc7be 1249 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1250 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1251 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1252 }
1253 }
c57f56b7 1254 }
1255 return kTRUE;
1256}
1257
a8739e8a 1258//______________________________________________________________________________
d2a409b2 1259Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1260{
1261// Writes one or more JDL's corresponding to findex. If findex is negative,
1262// all run numbers are considered in one go (jdl). For non-negative indices
1263// they correspond to the indices in the array fInputFiles.
1264 if (!fInputFiles) return kFALSE;
d3b18c4c 1265 TObject *os;
1266 TString workdir;
1267 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1268 workdir += fGridWorkingDir;
f47d5cb4 1269 TString stageName = "$2";
1270 if (fProductionMode) stageName = "$4";
1271 if (!fMergeDirName.IsNull()) {
1272 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1273 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1274 } else {
1275 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1276 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1277 }
d3b18c4c 1278 if (fProductionMode) {
a8739e8a 1279 TIter next(fInputFiles);
b3e07543 1280 while ((os=next())) {
d3b18c4c 1281 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1282 }
d3b18c4c 1283 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1284 } else {
1285 if (!fRunNumbers.Length() && !fRunRange[0]) {
1286 // One jdl with no parameters in case input data is specified by name.
1287 TIter next(fInputFiles);
1288 while ((os=next()))
1289 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1290 if (!fOutputSingle.IsNull())
1291 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1292 else {
1293 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1294 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1295 }
1296 } else {
1297 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1298 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1299 if (!fOutputSingle.IsNull()) {
1300 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1301 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1302 } else {
1303 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1304 }
1305 }
a8739e8a 1306 }
1307
a8739e8a 1308 // Generate the JDL as a string
1309 TString sjdl = fGridJDL->Generate();
0f389141 1310 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1311 // Final merge jdl
1312 if (!fMergeDirName.IsNull()) {
1313 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1314 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1315 } else {
1316 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1317 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1318 }
b3e07543 1319 TString sjdl2 = fMergingJDL->Generate();
1320 Int_t index, index1;
a8739e8a 1321 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1322 sjdl.ReplaceAll("(member", "\n (member");
1323 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1324 sjdl.ReplaceAll("{", "{\n ");
1325 sjdl.ReplaceAll("};", "\n};");
1326 sjdl.ReplaceAll("{\n \n", "{\n");
1327 sjdl.ReplaceAll("\n\n", "\n");
1328 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1329 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1330 sjdl1.ReplaceAll("(member", "\n (member");
1331 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1332 sjdl1.ReplaceAll("{", "{\n ");
1333 sjdl1.ReplaceAll("};", "\n};");
1334 sjdl1.ReplaceAll("{\n \n", "{\n");
1335 sjdl1.ReplaceAll("\n\n", "\n");
1336 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1337 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1338 sjdl2.ReplaceAll("(member", "\n (member");
1339 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1340 sjdl2.ReplaceAll("{", "{\n ");
1341 sjdl2.ReplaceAll("};", "\n};");
1342 sjdl2.ReplaceAll("{\n \n", "{\n");
1343 sjdl2.ReplaceAll("\n\n", "\n");
1344 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1345 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1346 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1347 index = sjdl.Index("JDLVariables");
1348 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1349 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1350 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1351 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1352 index = fJobTag.Index(":");
1353 if (index < 0) index = fJobTag.Length();
1354 TString jobTag = fJobTag;
f47d5cb4 1355 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1356 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1357 if (fProductionMode) {
1358 sjdl1.Prepend("# Generated merging jdl (production mode) \
1359 \n# $1 = full alien path to output directory to be merged \
1360 \n# $2 = train number \
1361 \n# $3 = production (like LHC10b) \
1362 \n# $4 = merging stage \
1363 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1364 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1365 sjdl2.Prepend("# Generated merging jdl \
1366 \n# $1 = full alien path to output directory to be merged \
1367 \n# $2 = train number \
1368 \n# $3 = production (like LHC10b) \
1369 \n# $4 = merging stage \
1370 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1371 } else {
1372 sjdl1.Prepend("# Generated merging jdl \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = merging stage \
03335324 1375 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1376 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1377 sjdl2.Prepend("# Generated merging jdl \
1378 \n# $1 = full alien path to output directory to be merged \
1379 \n# $2 = merging stage \
03335324 1380 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1381 }
0f389141 1382 index = sjdl1.Index("JDLVariables");
1383 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1384 index = sjdl2.Index("JDLVariables");
1385 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1386 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1387 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1388 index = sjdl2.Index("Split =");
1389 if (index>=0) {
1390 index1 = sjdl2.Index("\n", index);
1391 sjdl2.Remove(index, index1-index+1);
1392 }
1393 index = sjdl2.Index("SplitMaxInputFileNumber");
1394 if (index>=0) {
1395 index1 = sjdl2.Index("\n", index);
1396 sjdl2.Remove(index, index1-index+1);
1397 }
1398 index = sjdl2.Index("InputDataCollection");
1399 if (index>=0) {
1400 index1 = sjdl2.Index(";", index);
1401 sjdl2.Remove(index, index1-index+1);
1402 }
1403 index = sjdl2.Index("InputDataListFormat");
1404 if (index>=0) {
1405 index1 = sjdl2.Index("\n", index);
1406 sjdl2.Remove(index, index1-index+1);
1407 }
1408 index = sjdl2.Index("InputDataList");
1409 if (index>=0) {
1410 index1 = sjdl2.Index("\n", index);
1411 sjdl2.Remove(index, index1-index+1);
1412 }
f47d5cb4 1413 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1414 // Write jdl to file
a8739e8a 1415 ofstream out;
d2a409b2 1416 out.open(fJDLName.Data(), ios::out);
a8739e8a 1417 if (out.bad()) {
d3b18c4c 1418 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1419 return kFALSE;
1420 }
1421 out << sjdl << endl;
b3e07543 1422 out.close();
0f389141 1423 TString mergeJDLName = fExecutable;
1424 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1425 if (fMergeViaJDL) {
1426 ofstream out1;
1427 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1428 if (out1.bad()) {
d3b18c4c 1429 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1430 return kFALSE;
1431 }
1432 out1 << sjdl1 << endl;
b3e07543 1433 out1.close();
1434 ofstream out2;
1435 TString finalJDL = mergeJDLName;
1436 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1437 out2.open(finalJDL.Data(), ios::out);
1438 if (out2.bad()) {
1439 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1440 return kFALSE;
1441 }
1442 out2 << sjdl2 << endl;
1443 out2.close();
0f389141 1444 }
a8739e8a 1445
1446 // Copy jdl to grid workspace
1447 if (!copy) {
d3b18c4c 1448 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1449 } else {
d2a409b2 1450 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1451 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1452 TString finalJDL = mergeJDLName;
1453 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1454 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1455 if (fProductionMode) {
b5fe9cba 1456 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1457 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1458 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1459 }
d2a409b2 1460 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1461 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1462 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1463 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1464 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1465 if (fMergeViaJDL) {
b3e07543 1466 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1467 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1468 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1469 }
a8739e8a 1470 }
1471 return kTRUE;
1472}
1473
c57f56b7 1474//______________________________________________________________________________
5513444a 1475Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1476{
1477// Returns true if file exists.
5513444a 1478 if (!gGrid) return kFALSE;
2ea55496 1479 TString slfn = lfn;
1480 slfn.ReplaceAll("alien://","");
1481 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1482 if (!res) return kFALSE;
1483 TMap *map = dynamic_cast<TMap*>(res->At(0));
1484 if (!map) {
1485 delete res;
1486 return kFALSE;
1487 }
1488 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1489 if (!objs || !objs->GetString().Length()) {
1490 delete res;
1491 return kFALSE;
1492 }
1493 delete res;
1494 return kTRUE;
1495}
1496
923e2ca5 1497//______________________________________________________________________________
1498Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1499{
1500// Returns true if directory exists. Can be also a path.
1501 if (!gGrid) return kFALSE;
1502 // Check if dirname is a path
1503 TString dirstripped = dirname;
1504 dirstripped = dirstripped.Strip();
1505 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1506 TString dir = gSystem->BaseName(dirstripped);
1507 dir += "/";
1508 TString path = gSystem->DirName(dirstripped);
1509 TGridResult *res = gGrid->Ls(path, "-F");
1510 if (!res) return kFALSE;
1511 TIter next(res);
1512 TMap *map;
1513 TObject *obj;
1514 while ((map=dynamic_cast<TMap*>(next()))) {
1515 obj = map->GetValue("name");
1516 if (!obj) break;
1517 if (dir == obj->GetName()) {
1518 delete res;
1519 return kTRUE;
1520 }
1521 }
1522 delete res;
1523 return kFALSE;
1524}
1525
c57f56b7 1526//______________________________________________________________________________
a2f5fc01 1527void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1528{
1529// Check input data type.
a2f5fc01 1530 isCollection = kFALSE;
1531 isXml = kFALSE;
1532 useTags = kFALSE;
c57f56b7 1533 if (!gGrid) {
1534 Error("CheckDataType", "No connection to grid");
1535 return;
1536 }
a2f5fc01 1537 isCollection = IsCollection(lfn);
c57f56b7 1538 TString msg = "\n##### file: ";
1539 msg += lfn;
a2f5fc01 1540 if (isCollection) {
c57f56b7 1541 msg += " type: raw_collection;";
1542 // special treatment for collections
a2f5fc01 1543 isXml = kFALSE;
c57f56b7 1544 // check for tag files in the collection
1545 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1546 if (!res) {
1547 msg += " using_tags: No (unknown)";
23329835 1548 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1549 return;
1550 }
1551 const char* typeStr = res->GetKey(0, "origLFN");
1552 if (!typeStr || !strlen(typeStr)) {
1553 msg += " using_tags: No (unknown)";
23329835 1554 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1555 return;
1556 }
1557 TString file = typeStr;
a2f5fc01 1558 useTags = file.Contains(".tag");
1559 if (useTags) msg += " using_tags: Yes";
c57f56b7 1560 else msg += " using_tags: No";
23329835 1561 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1562 return;
1563 }
1564 TString slfn(lfn);
1565 slfn.ToLower();
a2f5fc01 1566 isXml = slfn.Contains(".xml");
1567 if (isXml) {
c57f56b7 1568 // Open xml collection and check if there are tag files inside
1569 msg += " type: xml_collection;";
1570 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1571 if (!coll) {
1572 msg += " using_tags: No (unknown)";
23329835 1573 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1574 return;
1575 }
1576 TMap *map = coll->Next();
1577 if (!map) {
1578 msg += " using_tags: No (unknown)";
23329835 1579 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1580 return;
1581 }
1582 map = (TMap*)map->GetValue("");
1583 TString file;
1584 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1585 useTags = file.Contains(".tag");
c57f56b7 1586 delete coll;
a2f5fc01 1587 if (useTags) msg += " using_tags: Yes";
c57f56b7 1588 else msg += " using_tags: No";
23329835 1589 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1590 return;
1591 }
a2f5fc01 1592 useTags = slfn.Contains(".tag");
c57f56b7 1593 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1594 else msg += " type: unknown file;";
a2f5fc01 1595 if (useTags) msg += " using_tags: Yes";
c57f56b7 1596 else msg += " using_tags: No";
23329835 1597 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1598}
1599
4e5c5506 1600//______________________________________________________________________________
1601void AliAnalysisAlien::EnablePackage(const char *package)
1602{
1603// Enables a par file supposed to exist in the current directory.
1604 TString pkg(package);
1605 pkg.ReplaceAll(".par", "");
1606 pkg += ".par";
1607 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1608 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1609 return;
1610 }
1611 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1612 Info("EnablePackage", "AliEn plugin will use .par packages");
1613 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1614 if (!fPackages) {
1615 fPackages = new TObjArray();
1616 fPackages->SetOwner();
1617 }
1618 fPackages->Add(new TObjString(pkg));
1619}
1620
3bdcb562 1621//______________________________________________________________________________
1622TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1623{
1624// Make a tree from files having the location specified in fFileForTestMode.
1625// Inspired from JF's CreateESDChain.
1626 if (fFileForTestMode.IsNull()) {
1627 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1628 return NULL;
1629 }
1630 if (gSystem->AccessPathName(fFileForTestMode)) {
1631 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1632 return NULL;
1633 }
1634 // Open the file
1635 ifstream in;
1636 in.open(fFileForTestMode);
1637 Int_t count = 0;
1638 // Read the input list of files and add them to the chain
1639 TString line;
1640 TChain *chain = new TChain(treeName);
1641 while (in.good())
1642 {
1643 in >> line;
1644 if (line.IsNull()) continue;
1645 if (count++ == fNtestFiles) break;
1646 TString esdFile(line);
1647 TFile *file = TFile::Open(esdFile);
1648 if (file) {
1649 if (!file->IsZombie()) chain->Add(esdFile);
1650 file->Close();
1651 } else {
1652 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1653 }
1654 }
1655 in.close();
1656 if (!chain->GetListOfFiles()->GetEntries()) {
1657 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1658 delete chain;
1659 return NULL;
1660 }
1661// chain->ls();
1662 return chain;
1663}
1664
d2a409b2 1665//______________________________________________________________________________
1666const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1667{
1668// Get job status for all jobs with jobid>jobidstart.
1669 static char mstatus[20];
1670 mstatus[0] = '\0';
1671 nrunning = 0;
1672 nwaiting = 0;
1673 nerror = 0;
1674 ndone = 0;
1675 TGridJobStatusList *list = gGrid->Ps("");
1676 if (!list) return mstatus;
1677 Int_t nentries = list->GetSize();
1678 TGridJobStatus *status;
1679 Int_t pid;
1680 for (Int_t ijob=0; ijob<nentries; ijob++) {
1681 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1682 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1683 if (pid<jobidstart) continue;
1684 if (pid == lastid) {
8ddad121 1685 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1686 }
1687 switch (status->GetStatus()) {
1688 case TGridJobStatus::kWAITING:
1689 nwaiting++; break;
1690 case TGridJobStatus::kRUNNING:
1691 nrunning++; break;
1692 case TGridJobStatus::kABORTED:
1693 case TGridJobStatus::kFAIL:
1694 case TGridJobStatus::kUNKNOWN:
1695 nerror++; break;
1696 case TGridJobStatus::kDONE:
1697 ndone++;
1698 }
1699 }
1700 list->Delete();
1701 delete list;
1702 return mstatus;
1703}
1704
c57f56b7 1705//______________________________________________________________________________
1706Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1707{
1708// Returns true if file is a collection. Functionality duplicated from
1709// TAlien::Type() because we don't want to directly depend on TAlien.
1710 if (!gGrid) {
1711 Error("IsCollection", "No connection to grid");
1712 return kFALSE;
1713 }
1714 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1715 if (!res) return kFALSE;
1716 const char* typeStr = res->GetKey(0, "type");
1717 if (!typeStr || !strlen(typeStr)) return kFALSE;
1718 if (!strcmp(typeStr, "collection")) return kTRUE;
1719 delete res;
1720 return kFALSE;
1721}
1722
fe2d7fc2 1723//______________________________________________________________________________
1724Bool_t AliAnalysisAlien::IsSingleOutput() const
1725{
1726// Check if single-ouput option is on.
1727 return (!fOutputSingle.IsNull());
1728}
1729
16a4353c 1730//______________________________________________________________________________
1731void AliAnalysisAlien::Print(Option_t *) const
1732{
1733// Print current plugin settings.
84fcd93f 1734 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1735 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1736 if (mgr && mgr->IsProofMode()) {
1737 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1738 if (TestBit(AliAnalysisGrid::kTest))
1739 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1740 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1741 if (!fProofDataSet.IsNull())
1742 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1743 if (fProofReset==1)
1744 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1745 if (fProofReset>1)
1746 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1747 if (!fRootVersionForProof.IsNull())
1748 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1749 else
1750 printf("= ROOT version requested________________________ default\n");
1751 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1752 if (!fAliRootMode.IsNull())
1753 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1754 if (fNproofWorkers)
1755 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1756 if (fNproofWorkersPerSlave)
1757 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1758 if (TestSpecialBit(kClearPackages))
1759 printf("= ClearPackages requested...\n");
1760 if (fIncludePath.Data())
1761 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1762 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1763 if (fPackages && fPackages->GetEntries()) {
1764 TIter next(fPackages);
1765 TObject *obj;
1766 TString list;
1767 while ((obj=next())) list += obj->GetName();
1768 printf("= Par files to be used: ________________________ %s\n", list.Data());
1769 }
1770 if (TestSpecialBit(kProofConnectGrid))
1771 printf("= Requested PROOF connection to grid\n");
1772 return;
1773 }
e1c22e21 1774 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1775 if (fOverwriteMode) {
1776 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1777 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1778 }
348be253 1779 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1780 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1781 printf("= Production mode:______________________________ %d\n", fProductionMode);
1782 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1783 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1784 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1785 if (fUser.Length())
84fcd93f 1786 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1787 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1788 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1789 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1790 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1791 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1792 if (fRunNumbers.Length())
84fcd93f 1793 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1794 if (fRunRange[0])
74013535 1795 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1796 if (!fRunRange[0] && !fRunNumbers.Length()) {
1797 TIter next(fInputFiles);
1798 TObject *obj;
1799 TString list;
1800 while ((obj=next())) list += obj->GetName();
84fcd93f 1801 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1802 }
1803 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1804 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1805 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1806 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1807 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1808 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1809 printf("=====================================================================\n");
1810 printf("= Job price: ___________________________________ %d\n", fPrice);
1811 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1812 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1813 if (fMaxInitFailed>0)
84fcd93f 1814 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1815 if (fMasterResubmitThreshold>0)
84fcd93f 1816 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1817 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1818 if (fNrunsPerMaster>0)
84fcd93f 1819 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1820 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1821 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1822 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1823 if (fArguments.Length())
84fcd93f 1824 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1825 if (fExecutableArgs.Length())
1826 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1827 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1828 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1829 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1830 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1831 if (fDatasetName)
84fcd93f 1832 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1833 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1834 if (fIncludePath.Data())
84fcd93f 1835 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1836 if (fCloseSE.Length())
84fcd93f 1837 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1838 if (fFriendChainName.Length())
84fcd93f 1839 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1840 if (fPackages && fPackages->GetEntries()) {
16a4353c 1841 TIter next(fPackages);
1842 TObject *obj;
1843 TString list;
1844 while ((obj=next())) list += obj->GetName();
84fcd93f 1845 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1846 }
1847}
1848
c57f56b7 1849//______________________________________________________________________________
1850void AliAnalysisAlien::SetDefaults()
1851{
1852// Set default values for everything. What cannot be filled will be left empty.
1853 if (fGridJDL) delete fGridJDL;
1854 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1855 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1856 fPrice = 1;
1857 fTTL = 30000;
1858 fSplitMaxInputFileNumber = 100;
1859 fMaxInitFailed = 0;
1860 fMasterResubmitThreshold = 0;
bb885a9e 1861 fNtestFiles = 10;
149d288c 1862 fNreplicas = 2;
a8739e8a 1863 fRunRange[0] = 0;
1864 fRunRange[1] = 0;
8e3bd06a 1865 fRunPrefix = "%d";
319593fb 1866 fNrunsPerMaster = 1;
16a4353c 1867 fMaxMergeFiles = 100;
c57f56b7 1868 fRunNumbers = "";
1869 fExecutable = "analysis.sh";
0a1c1f7f 1870 fExecutableCommand = "root -b -q";
c57f56b7 1871 fArguments = "";
631c0b05 1872 fExecutableArgs = "";
c57f56b7 1873 fAnalysisMacro = "myAnalysis.C";
1874 fAnalysisSource = "";
1875 fAdditionalLibs = "";
1876 fSplitMode = "se";
1877 fAPIVersion = "";
1878 fROOTVersion = "";
1879 fAliROOTVersion = "";
1880 fUser = ""; // Your alien user name
1881 fGridWorkingDir = "";
1882 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1883 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1884 fFriendChainName = "";
c57f56b7 1885 fGridOutputDir = "output";
b83f84f0 1886 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1887 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1888 fInputFormat = "xml-single";
1889 fJDLName = "analysis.jdl";
c6cb3634 1890 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1891 fMergeExcludes = "";
0f389141 1892 fMergeViaJDL = 0;
348be253 1893 SetUseCopy(kTRUE);
1894 SetCheckCopy(kTRUE);
149d288c 1895 SetDefaultOutputs(kTRUE);
e1c22e21 1896 fOverwriteMode = 1;
c57f56b7 1897}
1898
0f389141 1899//______________________________________________________________________________
b3e07543 1900Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1901{
b3e07543 1902// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1903 // First check if the result is already in the output directory.
1904 if (FileExists(Form("%s/%s",aliendir,filename))) {
1905 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1906 return kFALSE;
1907 }
b3e07543 1908 // Now check the last stage done.
1909 Int_t stage = 0;
1910 while (1) {
1911 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1912 stage++;
1913 }
1914 // Next stage of merging
1915 stage++;
1916 TString pattern = "*root_archive.zip";
1917 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1918 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1919 if (res) delete res;
1920 // Write standard output to file
1921 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1922 // Count the number of files inside
1923 ifstream ifile;
1924 ifile.open(Form("Stage_%d.xml",stage));
1925 if (!ifile.good()) {
1926 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1927 return kFALSE;
b3e07543 1928 }
1929 TString line;
1930 Int_t nfiles = 0;
1931 while (!ifile.eof()) {
1932 ifile >> line;
1933 if (line.Contains("/event")) nfiles++;
7c2cd90a 1934 }
b3e07543 1935 ifile.close();
1936 if (!nfiles) {
1937 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1938 return kFALSE;
1939 } else {
1940 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1941 }
b3e07543 1942 // Copy the file in the output directory
1943 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1944 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1945 // Check if this is the last stage to be done.
1946 Bool_t laststage = (nfiles<nperchunk);
1947 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1948 if (laststage) {
1949 printf("### Submiting final merging stage %d\n", stage);
1950 TString finalJDL = jdl;
1951 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1952 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1953 Int_t jobId = SubmitSingleJob(query);
1954 if (!jobId) return kFALSE;
1955 } else {
1956 printf("### Submiting merging stage %d\n", stage);
03335324 1957 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1958 Int_t jobId = SubmitSingleJob(query);
1959 if (!jobId) return kFALSE;
7c2cd90a 1960 }
b3e07543 1961 return kTRUE;
1962}
7c2cd90a 1963
1964//______________________________________________________________________________
1965Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1966{
1967// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1968 if (!gGrid) return 0;
1969 printf("=> %s ------> ",query);
1970 TGridResult *res = gGrid->Command(query);
1971 if (!res) return 0;
1972 TString jobId = res->GetKey(0,"jobId");
1973 delete res;
1974 if (jobId.IsNull()) {
1975 printf("submission failed. Reason:\n");
1976 gGrid->Stdout();
1977 gGrid->Stderr();
1978 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1979 return 0;
1980 }
1981 printf(" Job id: %s\n", jobId.Data());
1982 return atoi(jobId);
1983}
1984
1985//______________________________________________________________________________
b3e07543 1986Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 1987{
b3e07543 1988// Merge given output files from basedir. Basedir can be an alien output directory
1989// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
1990// files in a group (ignored for xml input). Merging can be done in stages:
1991// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
1992// stage=1 : works with an xml of all root_archive.zip in the output directory
1993// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 1994 TString outputFile = output;
0f389141 1995 TString command;
a2f5fc01 1996 TString outputChunk;
1997 TString previousChunk = "";
b3e07543 1998 TObjArray *listoffiles = new TObjArray();
1999// listoffiles->SetOwner();
a2f5fc01 2000 Int_t countChunk = 0;
2001 Int_t countZero = nmaxmerge;
0f389141 2002 Bool_t merged = kTRUE;
a2f5fc01 2003 Int_t index = outputFile.Index("@");
2004 if (index > 0) outputFile.Remove(index);
7c2cd90a 2005 TString inputFile = outputFile;
b3e07543 2006 TString sbasedir = basedir;
2007 if (sbasedir.Contains(".xml")) {
2008 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2009 nmaxmerge = 9999999;
2010 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2011 if (!coll) {
2012 ::Error("MergeOutput", "Input XML collection empty.");
2013 return kFALSE;
2014 }
2015 // Iterate grid collection
2016 while (coll->Next()) {
2017 TString fname = gSystem->DirName(coll->GetTURL());
2018 fname += "/";
2019 fname += inputFile;
2020 listoffiles->Add(new TNamed(fname.Data(),""));
2021 }
2022 } else {
2023 command = Form("find %s/ *%s", basedir, inputFile.Data());
2024 printf("command: %s\n", command.Data());
2025 TGridResult *res = gGrid->Command(command);
2026 if (!res) {
2027 ::Error("MergeOutput","No result for the find command\n");
2028 delete listoffiles;
2029 return kFALSE;
2030 }
2031 TIter nextmap(res);
2032 TMap *map = 0;
2033 while ((map=(TMap*)nextmap())) {
2034 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2035 if (!objs || !objs->GetString().Length()) {
2036 // Nothing found - skip this output
2037 delete res;
2038 delete listoffiles;
2039 return kFALSE;
2040 }
2041 listoffiles->Add(new TNamed(objs->GetName(),""));
2042 }
2043 delete res;
2044 }
2045 if (!listoffiles->GetEntries()) {
7c2cd90a 2046 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2047 delete listoffiles;
0f389141 2048 return kFALSE;
2049 }
2050
2051 TFileMerger *fm = 0;
b3e07543 2052 TIter next0(listoffiles);
2053 TObjArray *listoffilestmp = new TObjArray();
2054 listoffilestmp->SetOwner();
2055 TObject *nextfile;
2056 TString snextfile;
2057 // Keep only the files at upper level
2058 Int_t countChar = 0;
2059 while ((nextfile=next0())) {
2060 snextfile = nextfile->GetName();
2061 Int_t crtCount = snextfile.CountChar('/');
2062 if (nextfile == listoffiles->First()) countChar = crtCount;
2063 if (crtCount < countChar) countChar = crtCount;
2064 }
2065 next0.Reset();
2066 while ((nextfile=next0())) {
2067 snextfile = nextfile->GetName();
2068 Int_t crtCount = snextfile.CountChar('/');
2069 if (crtCount > countChar) {
2070 delete nextfile;
2071 continue;
2072 }
2073 listoffilestmp->Add(nextfile);
2074 }
2075 delete listoffiles;
2076 listoffiles = listoffilestmp; // Now contains 'good' files
2077 listoffiles->Print();
2078 TIter next(listoffiles);
7c2cd90a 2079 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2080 outputChunk = outputFile;
2081 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2082 // Check for existent temporary merge files
e1c22e21 2083 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2084 // Preserve old merging functionality for stage 0.
2085 if (stage==0) {
2086 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2087 while (1) {
2088 // Skip as many input files as in a chunk
2ea55496 2089 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2090 nextfile = next();
2091 if (!nextfile) {
2ea55496 2092 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2093 delete listoffiles;
2ea55496 2094 return kFALSE;
2095 }
b3e07543 2096 snextfile = nextfile->GetName();
7c2cd90a 2097 }
2098 outputChunk = outputFile;
2099 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2100 countChunk++;
2101 if (gSystem->AccessPathName(outputChunk)) continue;
2102 // Merged file with chunks up to <countChunk> found
2103 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2104 previousChunk = outputChunk;
2105 break;
2106 }
2107 }
2108 countZero = nmaxmerge;
2109
b3e07543 2110 while ((nextfile=next())) {
2111 snextfile = nextfile->GetName();
2ea55496 2112 // Loop 'find' results and get next LFN
7c2cd90a 2113 if (countZero == nmaxmerge) {
2114 // First file in chunk - create file merger and add previous chunk if any.
2115 fm = new TFileMerger(kFALSE);
2116 fm->SetFastMethod(kTRUE);
2117 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2118 outputChunk = outputFile;
2119 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2120 }
2121 // If last file found, put merged results in the output file
b3e07543 2122 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2123 // Add file to be merged and decrement chunk counter.
b3e07543 2124 fm->AddFile(snextfile);
7c2cd90a 2125 countZero--;
b3e07543 2126 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2127 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2128 // Nothing found - skip this output
2129 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2130 merged = kFALSE;
2131 break;
7c2cd90a 2132 }
2133 fm->OutputFile(outputChunk);
2134 // Merge the outputs, then go to next chunk
2135 if (!fm->Merge()) {
2136 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2137 merged = kFALSE;
2138 break;
7c2cd90a 2139 } else {
2140 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2141 gSystem->Unlink(previousChunk);
2142 }
b3e07543 2143 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2144 countChunk++;
2145 countZero = nmaxmerge;
2146 previousChunk = outputChunk;
0f389141 2147 }
0f389141 2148 }
b3e07543 2149 delete listoffiles;
2150 delete fm;
7c2cd90a 2151 return merged;
2152 }
2153 // Merging stage different than 0.
2154 // Move to the begining of the requested chunk.
7c2cd90a 2155 fm = new TFileMerger(kFALSE);
2156 fm->SetFastMethod(kTRUE);
b3e07543 2157 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2158 delete listoffiles;
7c2cd90a 2159 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2160 // Nothing found - skip this output
2161 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2162 delete fm;
2163 return kFALSE;
2164 }
b3e07543 2165 fm->OutputFile(outputFile);
7c2cd90a 2166 // Merge the outputs
2167 if (!fm->Merge()) {
2168 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2169 delete fm;
2170 return kFALSE;
2171 } else {
b3e07543 2172 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2173 }
7c2cd90a 2174 delete fm;
2175 return kTRUE;
0f389141 2176}
2177
c57f56b7 2178//______________________________________________________________________________
2179Bool_t AliAnalysisAlien::MergeOutputs()
2180{
2181// Merge analysis outputs existing in the AliEn space.
2182 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2183 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2184 if (!Connect()) {
2185 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2186 return kFALSE;
0f389141 2187 }
c9e8f7fd 2188 if (fMergeViaJDL) {
2189 if (!TestBit(AliAnalysisGrid::kMerge)) {
2190 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2191 return kFALSE;
2192 }
2193 if (fProductionMode) {
2194 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2195 return kFALSE;
2196 }
0f389141 2197 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2198 if (!SubmitMerging()) return kFALSE;
0f389141 2199 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2200 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2201 return kFALSE;
c57f56b7 2202 }
2203 // Get the output path
d2a409b2 2204 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2205 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2206 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2207 return kFALSE;
2208 }
2209 if (!fOutputFiles.Length()) {
2210 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2211 return kFALSE;
0f389141 2212 }
2213 // Check if fast read option was requested
e1c22e21 2214 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2215 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2216 if (fFastReadOption) {
d3339be3 2217 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2218 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2219 gEnv->SetValue("XNet.ConnectTimeout",50);
2220 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2221 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2222 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2223 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2224 }
e8b839ab 2225 // Make sure we change the temporary directory
2226 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2227 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2228 TIter next(list);
2229 TObjString *str;
a2f5fc01 2230 TString outputFile;
c57f56b7 2231 Bool_t merged = kTRUE;
2232 while((str=(TObjString*)next())) {
a2f5fc01 2233 outputFile = str->GetString();
2234 Int_t index = outputFile.Index("@");
2235 if (index > 0) outputFile.Remove(index);
2236 TString outputChunk = outputFile;
2237 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2238 // Skip already merged outputs
a2f5fc01 2239 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2240 if (fOverwriteMode) {
a2f5fc01 2241 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2242 gSystem->Unlink(outputFile);
2243 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2244 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2245 outputChunk.Data());
2246 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2247 }
2248 } else {
a2f5fc01 2249 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2250 continue;
2251 }
2252 } else {
a2f5fc01 2253 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2254 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2255 outputChunk.Data());
2256 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2257 }
2258 }
bb885a9e 2259 if (fMergeExcludes.Length() &&
a2f5fc01 2260 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2261 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2262 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2263 if (!merged) {
2264 Error("MergeOutputs", "Terminate() will NOT be executed");
2265 return kFALSE;
ff07ec61 2266 }
2267 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2268 if (fileOpened) fileOpened->Close();
c57f56b7 2269 }
0f389141 2270 return kTRUE;
c57f56b7 2271}
2272
bb885a9e 2273//______________________________________________________________________________
2274void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2275{
2276// Use the output files connected to output containers from the analysis manager
2277// rather than the files defined by SetOutputFiles
2278 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2279 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2280 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2281}
2282
149d288c 2283//______________________________________________________________________________
2284void AliAnalysisAlien::SetOutputFiles(const char *list)
2285{
2286// Manually set the output files list.
2287// Removes duplicates. Not allowed if default outputs are not disabled.
2288 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2289 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2290 return;
2291 }
2292 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2293 fOutputFiles = "";
2294 TString slist = list;
2295 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2296 TObjArray *arr = slist.Tokenize(" ");
2297 TObjString *os;
2298 TIter next(arr);
2299 TString sout;
2300 while ((os=(TObjString*)next())) {
2301 sout = os->GetString();
2302 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2303 if (fOutputFiles.Contains(sout)) continue;
2304 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2305 fOutputFiles += sout;
2306 }
2307 delete arr;
f790bc1b 2308}
149d288c 2309
2310//______________________________________________________________________________
2311void AliAnalysisAlien::SetOutputArchive(const char *list)
2312{
2313// Manually set the output archive list. Free text - you are on your own...
2314// Not allowed if default outputs are not disabled.
2315 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2316 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2317 return;
2318 }
2319 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2320 fOutputArchive = list;
2321}
2322
2323//______________________________________________________________________________
2324void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2325{
2326// Setting a prefered output SE is not allowed anymore.
2327 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2328}
2329
5513444a 2330//______________________________________________________________________________
2331Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2332{
2333// Start remote grid analysis.
3bdcb562 2334 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2335 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2336 if (!mgr || !mgr->IsInitialized()) {
2337 Error("StartAnalysis", "You need an initialized analysis manager for this");
2338 return kFALSE;
2339 }
2340 // Are we in PROOF mode ?
2341 if (mgr->IsProofMode()) {
90d50a8c 2342 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2343 if (fProofCluster.IsNull()) {
2344 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2345 return kFALSE;
2346 }
2347 if (fProofDataSet.IsNull() && !testMode) {
2348 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2349 return kFALSE;
2350 }
2351 // Set the needed environment
2352 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2353 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2354 if (fProofReset && !testMode) {
2355 if (fProofReset==1) {
2356 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2357 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2358 } else {
2359 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2360 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2361 }
2362 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2363 return kFALSE;
2364 }
2365 // Do we need to change the ROOT version ? The success of this cannot be checked.
2366 if (!fRootVersionForProof.IsNull() && !testMode) {
2367 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2368 fProofCluster.Data(), fRootVersionForProof.Data()));
2369 }
2370 // Connect to PROOF and check the status
2371 Long_t proof = 0;
27734f0e 2372 TString sworkers;
2373 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2374 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2375 if (!testMode) {
27734f0e 2376 if (!sworkers.IsNull())
2377 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2378 else
2379 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2380 } else {
2381 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2382 if (!proof) {
2383 Error("StartAnalysis", "Could not start PROOF in test mode");
2384 return kFALSE;
2385 }
2386 }
2387 if (!proof) {
2388 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2389 return kFALSE;
2390 }
27734f0e 2391 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2392 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2393 // Is dataset existing ?
2394 if (!testMode) {
2395 TString dataset = fProofDataSet;
2396 Int_t index = dataset.Index("#");
2397 if (index>=0) dataset.Remove(index);
830acc4c 2398// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2399// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2400// return kFALSE;
2401// }
2402// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2403 }
2404 // Is ClearPackages() needed ?
2405 if (TestSpecialBit(kClearPackages)) {
2406 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2407 gROOT->ProcessLine("gProof->ClearPackages();");
2408 }
2409 // Is a given aliroot mode requested ?
2410 TList optionsList;
1ac846e9 2411 TString parLibs;
830acc4c 2412 if (!fAliRootMode.IsNull()) {
3bdcb562 2413 TString alirootMode = fAliRootMode;
2414 if (alirootMode == "default") alirootMode = "";
2415 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2416 optionsList.SetOwner();
2417 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2418 // Check the additional libs to be loaded
2419 TString extraLibs;
1ac846e9 2420 Bool_t parMode = kFALSE;
3bdcb562 2421 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2422 // Parse the extra libs for .so
2423 if (fAdditionalLibs.Length()) {
2424 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2425 TIter next(list);
2426 TObjString *str;
1ac846e9 2427 while((str=(TObjString*)next())) {
2428 if (str->GetString().Contains(".so")) {
2429 if (parMode) {
2430 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2431 break;
2432 }
2433 TString stmp = str->GetName();
2434 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2435 stmp.ReplaceAll(".so","");
2436 if (!extraLibs.IsNull()) extraLibs += ":";
2437 extraLibs += stmp;
2438 continue;
2439 }
2440 if (str->GetString().Contains(".par")) {
2441 // The first par file found in the list will not allow any further .so
2442 parMode = kTRUE;
2443 if (!parLibs.IsNull()) parLibs += ":";
2444 parLibs += str->GetName();
2445 continue;
2446 }
3bdcb562 2447 }
2448 if (list) delete list;
2449 }
2450 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2451 // Check extra includes
2452 if (!fIncludePath.IsNull()) {
2453 TString includePath = fIncludePath;
2454 includePath.ReplaceAll(" ",":");
8ddad121 2455 includePath.ReplaceAll("$ALICE_ROOT","");
2456 includePath.ReplaceAll("${ALICE_ROOT}","");
2457 includePath.ReplaceAll("-I","");
830acc4c 2458 includePath.Strip(TString::kTrailing, ':');
2459 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2460 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2461 }
27734f0e 2462 // Check if connection to grid is requested
2463 if (TestSpecialBit(kProofConnectGrid))
2464 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2465 // Enable AliRoot par
2466 if (testMode) {
2467 // Enable proof lite package
2468 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2469 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2470 TNamed *obj = (TNamed*)optionsList.At(i);
2471 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2472 }
2473 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2474 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2475 Info("StartAnalysis", "AliRootProofLite enabled");
2476 } else {
2477 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2478 return kFALSE;
2479 }
2480 } else {
8ddad121 2481 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2482 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2483 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2484 return kFALSE;
2485 }
27734f0e 2486 }
1ac846e9 2487 // Enable first par files from fAdditionalLibs
2488 if (!parLibs.IsNull()) {
2489 TObjArray *list = parLibs.Tokenize(":");
2490 TIter next(list);
2491 TObjString *package;
2492 while((package=(TObjString*)next())) {
40501bb7 2493 TString spkg = package->GetName();
2494 spkg.ReplaceAll(".par", "");
2495 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2496 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2497 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2498 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2499 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2500 return kFALSE;
2501 }
2502 } else {
2503 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2504 return kFALSE;
2505 }
2506 }
2507 if (list) delete list;
2508 }
3bdcb562 2509 } else {
2510 if (fAdditionalLibs.Contains(".so") && !testMode) {
2511 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2512 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2513 return kFALSE;
2514 }
2515 }
2516 // Enable par files if requested
2517 if (fPackages && fPackages->GetEntries()) {
2518 TIter next(fPackages);
2519 TObject *package;
2520 while ((package=next())) {
1ac846e9 2521 // Skip packages already enabled
2522 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2523 TString spkg = package->GetName();
2524 spkg.ReplaceAll(".par", "");
2525 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2526 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2527 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2528 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2529 return kFALSE;
2530 }
2531 } else {
2532 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2533 return kFALSE;
2534 }
2535 }
2536 }
2537 // Do we need to load analysis source files ?
2538 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2539 if (fAnalysisSource.Length()) {
2540 TObjArray *list = fAnalysisSource.Tokenize(" ");
2541 TIter next(list);
2542 TObjString *str;
2543 while((str=(TObjString*)next())) {
2544 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2545 }
2546 if (list) delete list;
2547 }
830acc4c 2548 if (testMode) {
2549 // Register dataset to proof lite.
2550 if (fFileForTestMode.IsNull()) {
2551 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2552 return kFALSE;
2553 }
2554 if (gSystem->AccessPathName(fFileForTestMode)) {
2555 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2556 return kFALSE;
2557 }
2558 TFileCollection *coll = new TFileCollection();
2559 coll->AddFromFile(fFileForTestMode);
8ddad121 2560 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2561 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2562 }
3bdcb562 2563 return kTRUE;
2564 }
c57f56b7 2565
43da816a 2566 // Check if output files have to be taken from the analysis manager
2567 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2568 // Add output files and AOD files
2569 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2570 // Add extra files registered to the analysis manager
f790bc1b 2571 TString extra = GetListOfFiles("ext");
2572 if (!extra.IsNull()) {
149d288c 2573 extra.ReplaceAll(".root", "*.root");
f790bc1b 2574 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2575 fOutputFiles += extra;
84fcd93f 2576 }
149d288c 2577 // Compose the output archive.
b83f84f0 2578 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2579 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2580 }
f7b1cbc2 2581// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2582 if (TestBit(AliAnalysisGrid::kOffline)) {
2583 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2584 \n there nor any job run. You can revise the JDL and analysis \
2585 \n macro then run the same in \"submit\" mode.");
2586 } else if (TestBit(AliAnalysisGrid::kTest)) {
2587 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2588 \n dataset.");
2589 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2590 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2591 \n space and job submitted.");
2592 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2593 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2594 if (fMergeViaJDL) CheckInputData();
5513444a 2595 return kTRUE;
c57f56b7 2596 } else {
2597 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2598 }
2599
348be253 2600 Print();
c57f56b7 2601 if (!Connect()) {
2602 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2603 return kFALSE;
16a4353c 2604 }
d3b18c4c 2605 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2606 if (!CheckInputData()) {
2607 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2608 return kFALSE;
c57f56b7 2609 }
d3339be3 2610 if (!CreateDataset(fDataPattern)) {
2611 TString serror;
2612 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2613 if (fRunNumbers.Length()) serror = "run numbers";
2614 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2615 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2616 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2617 return kFALSE;
2618 }
c57f56b7 2619 WriteAnalysisFile();
2620 WriteAnalysisMacro();
2621 WriteExecutable();
2622 WriteValidationScript();
0f389141 2623 if (fMergeViaJDL) {
2624 WriteMergingMacro();
2625 WriteMergeExecutable();
2626 WriteValidationScript(kTRUE);
2627 }
5513444a 2628 if (!CreateJDL()) return kFALSE;
2629 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2630 if (testMode) {
c57f56b7 2631 // Locally testing the analysis
2632 Info("StartAnalysis", "\n_______________________________________________________________________ \
2633 \n Running analysis script in a daughter shell as on a worker node \
2634 \n_______________________________________________________________________");
149d288c 2635 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2636 TIter next(list);
2637 TObjString *str;
a2f5fc01 2638 TString outputFile;
c57f56b7 2639 while((str=(TObjString*)next())) {
a2f5fc01 2640 outputFile = str->GetString();
2641 Int_t index = outputFile.Index("@");
2642 if (index > 0) outputFile.Remove(index);
2643 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2644 }
2645 delete list;
2646 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2647 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2648// gSystem->Exec("cat stdout");
5513444a 2649 return kFALSE;
c57f56b7 2650 }
5513444a 2651 // Check if submitting is managed by LPM manager
a3e84053 2652 if (fProductionMode) {
5513444a 2653 TString prodfile = fJDLName;
2654 prodfile.ReplaceAll(".jdl", ".prod");
2655 WriteProductionFile(prodfile);
2656 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2657 return kFALSE;
2658 }
a8739e8a 2659 // Submit AliEn job(s)
d2a409b2 2660 gGrid->Cd(fGridOutputDir);
a8739e8a 2661 TGridResult *res;
c57f56b7 2662 TString jobID = "";
d2a409b2 2663 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2664 // Submit a given xml or a set of runs
a8739e8a 2665 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2666 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2667 if (res) {
2668 const char *cjobId = res->GetKey(0,"jobId");
2669 if (!cjobId) {
a03be957 2670 gGrid->Stdout();
2671 gGrid->Stderr();
a8739e8a 2672 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2673 return kFALSE;
a8739e8a 2674 } else {
2675 Info("StartAnalysis", "\n_______________________________________________________________________ \
2676 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2677 \n_______________________________________________________________________",
2678 fJDLName.Data(), cjobId);
2679 jobID = cjobId;
2680 }
2681 delete res;
a03be957 2682 } else {
2683 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2684 return kFALSE;
a8739e8a 2685 }
2686 } else {
d2a409b2 2687 // Submit for a range of enumeration of runs.
a03be957 2688 if (!Submit()) return kFALSE;
c57f56b7 2689 }
a8739e8a 2690
c57f56b7 2691 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2692 \n You may exit at any time and terminate the job later using the option <terminate> \
2693 \n ##################################################################################", jobID.Data());
bb885a9e 2694 gSystem->Exec("aliensh");
5513444a 2695 return kTRUE;
c57f56b7 2696}
2697
f790bc1b 2698//______________________________________________________________________________
2699const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2700{
2701// Get a comma-separated list of output files of the requested type.
2702// Type can be (case unsensitive):
2703// aod - list of aod files (std, extensions and filters)
2704// out - list of output files connected to containers (but not aod's or extras)
2705// ext - list of extra files registered to the manager
2706// ter - list of files produced in terminate
2707 static TString files;
2708 files = "";
2709 TString stype = type;
2710 stype.ToLower();
2711 TString aodfiles, extra;
2712 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2713 if (!mgr) {
2714 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2715 return files.Data();
2716 }
2717 if (mgr->GetOutputEventHandler()) {
2718 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2719 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2720 if (!extraaod.IsNull()) {
2721 aodfiles += ",";
2722 aodfiles += extraaod;
2723 }
2724 }
2725 if (stype.Contains("aod")) {
2726 files = aodfiles;
2727 if (stype == "aod") return files.Data();
2728 }
2729 // Add output files that are not in the list of AOD files
2730 TString outputfiles = "";
2731 TIter next(mgr->GetOutputs());
2732 AliAnalysisDataContainer *output;
2733 const char *filename = 0;
2734 while ((output=(AliAnalysisDataContainer*)next())) {
2735 filename = output->GetFileName();
2736 if (!(strcmp(filename, "default"))) continue;
2737 if (outputfiles.Contains(filename)) continue;
2738 if (aodfiles.Contains(filename)) continue;
2739 if (!outputfiles.IsNull()) outputfiles += ",";
2740 outputfiles += filename;
2741 }
2742 if (stype.Contains("out")) {
2743 if (!files.IsNull()) files += ",";
2744 files += outputfiles;
2745 if (stype == "out") return files.Data();
2746 }
2747 // Add extra files registered to the analysis manager
2748 TString sextra;
2749 extra = mgr->GetExtraFiles();
2750 if (!extra.IsNull()) {
2751 extra.Strip();
2752 extra.ReplaceAll(" ", ",");
2753 TObjArray *fextra = extra.Tokenize(",");
2754 TIter nextx(fextra);
2755 TObject *obj;
2756 while ((obj=nextx())) {
2757 if (aodfiles.Contains(obj->GetName())) continue;
2758 if (outputfiles.Contains(obj->GetName())) continue;
2759 if (sextra.Contains(obj->GetName())) continue;
2760 if (!sextra.IsNull()) sextra += ",";
2761 sextra += obj->GetName();
2762 }
2763 delete fextra;
2764 if (stype.Contains("ext")) {
2765 if (!files.IsNull()) files += ",";
2766 files += sextra;
2767 }
2768 }
2769 if (stype == "ext") return files.Data();
2770 TString termfiles;
2771 if (!fTerminateFiles.IsNull()) {
2772 fTerminateFiles.Strip();
2773 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2774 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2775 TIter nextx(fextra);
2776 TObject *obj;
2777 while ((obj=nextx())) {
2778 if (aodfiles.Contains(obj->GetName())) continue;
2779 if (outputfiles.Contains(obj->GetName())) continue;
2780 if (termfiles.Contains(obj->GetName())) continue;
2781 if (sextra.Contains(obj->GetName())) continue;
2782 if (!termfiles.IsNull()) termfiles += ",";
2783 termfiles += obj->GetName();
2784 }
2785 delete fextra;
2786 }
2787 if (stype.Contains("ter")) {
2788 if (!files.IsNull() && !termfiles.IsNull()) {
2789 files += ",";
2790 files += termfiles;
2791 }
2792 }
2793 return files.Data();
2794}
2795
d2a409b2 2796//______________________________________________________________________________
a03be957 2797Bool_t AliAnalysisAlien::Submit()
d2a409b2 2798{
2799// Submit all master jobs.
2800 Int_t nmasterjobs = fInputFiles->GetEntries();
2801 Long_t tshoot = gSystem->Now();
a03be957 2802 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2803 while (fNsubmitted < nmasterjobs) {
2804 Long_t now = gSystem->Now();
2805 if ((now-tshoot)>30000) {
2806 tshoot = now;
a03be957 2807 if (!SubmitNext()) return kFALSE;
d2a409b2 2808 }
2809 }
a03be957 2810 return kTRUE;
d2a409b2 2811}
2812
0f389141 2813//______________________________________________________________________________
a03be957 2814Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2815{
2816// Submit all merging jobs.
2817 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2818 gGrid->Cd(fGridOutputDir);
2819 TString mergeJDLName = fExecutable;
2820 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2821 if (!fInputFiles) {
2822 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2823 return kFALSE;
2824 }
0f389141 2825 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2826 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2827 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2828 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2829 if (fOutputToRunNo) {
2830 // The output directory is the run number
2831 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2832 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2833 } else {
2834 // The output directory is the master number in 3 digits format
2835 printf("### Submitting merging job for master <%03d>\n", i);
2836 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2837 }
2838 // Check now the number of merging stages.
2b222f7c 2839 TObjArray *list = fOutputFiles.Tokenize(",");
2840 TIter next(list);
2841 TObjString *str;
2842 TString outputFile;
2843 while((str=(TObjString*)next())) {
2844 outputFile = str->GetString();
2845 Int_t index = outputFile.Index("@");
2846 if (index > 0) outputFile.Remove(index);
2847 if (!fMergeExcludes.Contains(outputFile)) break;
2848 }
2849 delete list;
b3e07543 2850 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2851 if (!done && (i==ntosubmit-1)) return kFALSE;
0f389141 2852 }
a03be957 2853 if (!ntosubmit) return kTRUE;
9f020952 2854 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2855 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2856 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2857 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2858 \n ################################################################################################################");
0f389141 2859 gSystem->Exec("aliensh");
a03be957 2860 return kTRUE;
0f389141 2861}
2862
d2a409b2 2863//______________________________________________________________________________
a03be957 2864Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2865{
08d5b699 2866// Submit next bunch of master jobs if the queue is free. The first master job is
2867// submitted right away, while the next will not be unless the previous was split.
2868// The plugin will not submit new master jobs if there are more that 500 jobs in
2869// waiting phase.
d2a409b2 2870 static Bool_t iscalled = kFALSE;
2871 static Int_t firstmaster = 0;
2872 static Int_t lastmaster = 0;
2873 static Int_t npermaster = 0;
a03be957 2874 if (iscalled) return kTRUE;
d2a409b2 2875 iscalled = kTRUE;
2876 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2877 Int_t ntosubmit = 0;
2878 TGridResult *res;
2879 TString jobID = "";
3bdcb562 2880 Int_t nmasterjobs = fInputFiles->GetEntries();
2881 if (!fNsubmitted) {
2882 ntosubmit = 1;
2883 if (!IsUseSubmitPolicy()) {
90d50a8c 2884 if (nmasterjobs>5)
3bdcb562 2885 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2886 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2887 ntosubmit = nmasterjobs;
2888 }
2889 } else {
d2a409b2 2890 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2891 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2892 // If last master not split, just return
a03be957 2893 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2894 // No more than 100 waiting jobs
08d5b699 2895 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2896 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2897 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2898 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2899 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2900 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2901 }
d2a409b2 2902 for (Int_t i=0; i<ntosubmit; i++) {
2903 // Submit for a range of enumeration of runs.
a03be957 2904 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2905 TString query;
cd11251e 2906 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2907 runOutDir.ReplaceAll(".xml", "");
2908 if (fOutputToRunNo)
2909 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2910 else
2911 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2912 printf("********* %s\n",query.Data());
d2a409b2 2913 res = gGrid->Command(query);
2914 if (res) {
98ca124f 2915 TString cjobId1 = res->GetKey(0,"jobId");
2916 if (!cjobId1.Length()) {
d2a409b2 2917 iscalled = kFALSE;
a03be957 2918 gGrid->Stdout();
2919 gGrid->Stderr();
2920 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2921 return kFALSE;
d2a409b2 2922 } else {
2923 Info("StartAnalysis", "\n_______________________________________________________________________ \
2924 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2925 \n_______________________________________________________________________",
98ca124f 2926 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2927 jobID += cjobId1;
2928 jobID += " ";
98ca124f 2929 lastmaster = cjobId1.Atoi();
d2a409b2 2930 if (!firstmaster) firstmaster = lastmaster;
2931 fNsubmitted++;
2932 }
2933 delete res;
a03be957 2934 } else {
2935 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2936 return kFALSE;
d2a409b2 2937 }
2938 }
2939 iscalled = kFALSE;
a03be957 2940 return kTRUE;
d2a409b2 2941}
2942
c57f56b7 2943//______________________________________________________________________________
2944void AliAnalysisAlien::WriteAnalysisFile()
2945{
f10e8481 2946// Write current analysis manager into the file <analysisFile>
2947 TString analysisFile = fExecutable;
2948 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2949 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2950 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2951 if (!mgr || !mgr->IsInitialized()) {
2952 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2953 return;
2954 }
2955 // Check analysis type
2956 TObject *handler;
2957 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2958 handler = (TObject*)mgr->GetInputEventHandler();
2959 if (handler) {
2960 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2961 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2962 }
2963 TDirectory *cdir = gDirectory;
f10e8481 2964 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2965 if (file) {
e16a394c 2966 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2967 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2968 // Unless merging makes no sense
2969 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2970 mgr->Write();
2971 delete file;
fe2d7fc2 2972 // Enable termination for local jobs
2973 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2974 }
2975 if (cdir) cdir->cd();
f10e8481 2976 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2977 }
2978 Bool_t copy = kTRUE;
d3b18c4c 2979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 2980 if (copy) {
2981 CdWork();
2982 TString workdir = gGrid->GetHomeDirectory();
2983 workdir += fGridWorkingDir;
d3b18c4c 2984 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 2985 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
2986 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 2987 }
2988}
2989
2990//______________________________________________________________________________
2991void AliAnalysisAlien::WriteAnalysisMacro()
2992{
2993// Write the analysis macro that will steer the analysis in grid mode.
2994 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2995 ofstream out;
2996 out.open(fAnalysisMacro.Data(), ios::out);
2997 if (!out.good()) {
2998 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2999 return;
3000 }
5b273635 3001 Bool_t hasSTEERBase = kFALSE;
3002 Bool_t hasESD = kFALSE;
3003 Bool_t hasAOD = kFALSE;
3004 Bool_t hasANALYSIS = kFALSE;
3005 Bool_t hasANALYSISalice = kFALSE;
3006 Bool_t hasCORRFW = kFALSE;
c57f56b7 3007 TString func = fAnalysisMacro;
3008 TString type = "ESD";
3009 TString comment = "// Analysis using ";
3010 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3011 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3012 type = "AOD";
3013 comment += "AOD";
3014 }
0df6ccf2 3015 if (type!="AOD" && fFriendChainName!="") {
3016 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3017 return;
3018 }
c57f56b7 3019 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3020 else comment += " data";
3021 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3022 func.ReplaceAll(".C", "");
3023 out << "void " << func.Data() << "()" << endl;
3024 out << "{" << endl;
3025 out << comment.Data() << endl;
3026 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3027 out << " TStopwatch timer;" << endl;
3028 out << " timer.Start();" << endl << endl;
13f0c00b 3029 // Change temp directory to current one
3030 out << "// Set temporary merging directory to current one" << endl;
3031 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
b3e07543 3032 // Reset existing include path
3033 out << "// Reset existing include path and add current directory first in the search" << endl;
3034 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3035 if (!fExecutableCommand.Contains("aliroot")) {
3036 out << "// load base root libraries" << endl;
3037 out << " gSystem->Load(\"libTree\");" << endl;
3038 out << " gSystem->Load(\"libGeom\");" << endl;
3039 out << " gSystem->Load(\"libVMC\");" << endl;
3040 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3041 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3042 }
d5c6455a 3043 if (fAdditionalRootLibs.Length()) {
47a4137d 3044 // in principle libtree /lib geom libvmc etc. can go into this list, too
3045 out << "// Add aditional libraries" << endl;
3046 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3047 TIter next(list);
3048 TObjString *str;
3049 while((str=(TObjString*)next())) {
3050 if (str->GetString().Contains(".so"))
3051 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3052 }
47a4137d 3053 if (list) delete list;
d5c6455a 3054 }
57377eb5 3055 out << "// Load analysis framework libraries" << endl;
652561d5 3056 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3057 if (!fPackages) {
f790bc1b 3058 if (!fExecutableCommand.Contains("aliroot")) {
3059 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3060 out << " gSystem->Load(\"libESD\");" << endl;
3061 out << " gSystem->Load(\"libAOD\");" << endl;
3062 }
4e5c5506 3063 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3064 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3065 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3066 } else {
4e5c5506 3067 TIter next(fPackages);
3068 TObject *obj;
57377eb5 3069 TString pkgname;
57377eb5 3070 while ((obj=next())) {
3071 pkgname = obj->GetName();
4478e6f1 3072 if (pkgname == "STEERBase" ||
3073 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3074 if (pkgname == "ESD" ||
3075 pkgname == "ESD.par") hasESD = kTRUE;
3076 if (pkgname == "AOD" ||
3077 pkgname == "AOD.par") hasAOD = kTRUE;
3078 if (pkgname == "ANALYSIS" ||
3079 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3080 if (pkgname == "ANALYSISalice" ||
3081 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3082 if (pkgname == "CORRFW" ||
3083 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3084 }
3085 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3086 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3087 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3088 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3089 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3090 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3091 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3092 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3093 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3094 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3095 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3096 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3097 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3098 out << "// Compile other par packages" << endl;
3099 next.Reset();
fcc9bb6f 3100 while ((obj=next())) {
3101 pkgname = obj->GetName();
4478e6f1 3102 if (pkgname == "STEERBase" ||
3103 pkgname == "STEERBase.par" ||
3104 pkgname == "ESD" ||
3105 pkgname == "ESD.par" ||
3106 pkgname == "AOD" ||
3107 pkgname == "AOD.par" ||
3108 pkgname == "ANALYSIS" ||
3109 pkgname == "ANALYSIS.par" ||
3110 pkgname == "ANALYSISalice" ||
3111 pkgname == "ANALYSISalice.par" ||
3112 pkgname == "CORRFW" ||
3113 pkgname == "CORRFW.par") continue;
5b273635 3114 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3115 }
4e5c5506 3116 }
b3e07543 3117 out << "// include path" << endl;
3118 // Get the include path from the interpreter and remove entries pointing to AliRoot
3119 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3120 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3121 out << " TIter nextpath(listpaths);" << endl;
3122 out << " TObjString *pname;" << endl;
3123 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3124 out << " TString current = pname->GetName();" << endl;
3125 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3126 out << " gSystem->AddIncludePath(current);" << endl;
3127 out << " }" << endl;
3128 out << " if (listpaths) delete listpaths;" << endl;
3129 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3130 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3131 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3132 if (fAdditionalLibs.Length()) {
3133 out << "// Add aditional AliRoot libraries" << endl;
3134 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3135 TIter next(list);
3136 TObjString *str;
3137 while((str=(TObjString*)next())) {
3138 if (str->GetString().Contains(".so"))
3139 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3140 if (str->GetString().Contains(".par"))
3141 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3142 }
3143 if (list) delete list;
3144 }
3145 out << endl;
c57f56b7 3146 out << "// analysis source to be compiled at runtime (if any)" << endl;
3147 if (fAnalysisSource.Length()) {
3148 TObjArray *list = fAnalysisSource.Tokenize(" ");
3149 TIter next(list);
3150 TObjString *str;
3151 while((str=(TObjString*)next())) {
3152 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3153 }
3154 if (list) delete list;
3155 }
3156 out << endl;
b3e07543 3157// out << " printf(\"Currently load libraries:\\n\");" << endl;
3158// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3159 if (fFastReadOption) {
d3339be3 3160 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3161 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3162 out << "// fast xrootd reading enabled" << endl;
3163 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3164 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3165 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3166 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3167 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3168 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3169 }
c57f56b7 3170 out << "// connect to AliEn and make the chain" << endl;
3171 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3172 out << "// read the analysis manager from file" << endl;
f10e8481 3173 TString analysisFile = fExecutable;
3174 analysisFile.ReplaceAll(".sh", ".root");
3175 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3176 out << " if (!file) return;" << endl;
3177 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3178 out << " AliAnalysisManager *mgr = 0;" << endl;
3179 out << " TKey *key;" << endl;
3180 out << " while ((key=(TKey*)nextkey())) {" << endl;
3181 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3182 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3183 out << " };" << endl;
3184 out << " if (!mgr) {" << endl;
205b201f 3185 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3186 out << " return;" << endl;
3187 out << " }" << endl << endl;
3188 out << " mgr->PrintStatus();" << endl;
52b6a92b 3189 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3190 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3191 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3192 } else {
9a8288ff 3193 if (TestBit(AliAnalysisGrid::kTest))
3194 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3195 else
3196 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3197 }
3198 }
242accb2 3199 if (IsUsingTags()) {
3200 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3201 } else {
3202 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3203 }
c57f56b7 3204 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3205 out << " timer.Stop();" << endl;
3206 out << " timer.Print();" << endl;
c57f56b7 3207 out << "}" << endl << endl;
3208 if (IsUsingTags()) {
3209 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3210 out << "{" << endl;
3211 out << "// Create a chain using tags from the xml file." << endl;
3212 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3213 out << " if (!coll) {" << endl;
3214 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3215 out << " return NULL;" << endl;
3216 out << " }" << endl;
3217 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3218 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3219 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3220 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3221 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3222 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3223 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3224 out << " // Check if the cuts configuration file was provided" << endl;
3225 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3226 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3227 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3228 out << " }" << endl;
0df6ccf2 3229 if (fFriendChainName=="") {
3230 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3231 } else {
3232 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3233 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3234 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3235 }
c57f56b7 3236 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3237 out << " chain->ls();" << endl;
3238 out << " return chain;" << endl;
fcc9bb6f 3239 out << "}" << endl << endl;
c57f56b7 3240 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3241 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3242 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3243 msg += " AliLHCTagCuts *lhcCuts,\n";
3244 msg += " AliDetectorTagCuts *detCuts,\n";
3245 msg += " AliEventTagCuts *evCuts)";
23329835 3246 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3247 }
0df6ccf2 3248 }
3249 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3250 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3251 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3252 out << "{" << endl;
3253 out << "// Create a chain using url's from xml file" << endl;
242accb2 3254 out << " TString filename;" << endl;
3255 out << " Int_t run = 0;" << endl;
c57f56b7 3256 out << " TString treename = type;" << endl;
3257 out << " treename.ToLower();" << endl;
3258 out << " treename += \"Tree\";" << endl;
e02fee64 3259 out << " printf(\"***************************************\\n\");" << endl;
3260 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3261 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3262 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3263 out << " if (!coll) {" << endl;
3264 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3265 out << " return NULL;" << endl;
3266 out << " }" << endl;
242accb2 3267 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3268 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3269 if(fFriendChainName!="") {
3270 out << " TChain *chainFriend = new TChain(treename);" << endl;
3271 }
c57f56b7 3272 out << " coll->Reset();" << endl;
0df6ccf2 3273 out << " while (coll->Next()) {" << endl;
242accb2 3274 out << " filename = coll->GetTURL("");" << endl;
3275 out << " if (mgr) {" << endl;
3276 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3277 out << " if (nrun && nrun != run) {" << endl;
3278 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3279 out << " mgr->SetRunFromPath(nrun);" << endl;
3280 out << " run = nrun;" << endl;
3281 out << " }" << endl;
3282 out << " }" << endl;
3283 out << " chain->Add(filename);" << endl;
0df6ccf2 3284 if(fFriendChainName!="") {
3285 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3286 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3287 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3288 out << " chainFriend->Add(fileFriend.Data());" << endl;
3289 }
3290 out << " }" << endl;
c57f56b7 3291 out << " if (!chain->GetNtrees()) {" << endl;
3292 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3293 out << " return NULL;" << endl;
3294 out << " }" << endl;
0df6ccf2 3295 if(fFriendChainName!="") {
3296 out << " chain->AddFriend(chainFriend);" << endl;
3297 }
c57f56b7 3298 out << " return chain;" << endl;
fcc9bb6f 3299 out << "}" << endl << endl;
c57f56b7 3300 }
5b273635 3301 if (hasANALYSISalice) {
3302 out <<"//________________________________________________________________________________" << endl;
3303 out << "Bool_t SetupPar(const char *package) {" << endl;
3304 out << "// Compile the package and set it up." << endl;
3305 out << " TString pkgdir = package;" << endl;
3306 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3307 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3308 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3309 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3310 out << " // Check for BUILD.sh and execute" << endl;
3311 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3312 out << " printf(\"*******************************\\n\");" << endl;
3313 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3314 out << " printf(\"*******************************\\n\");" << endl;
3315 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3316 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3317 out << " gSystem->ChangeDirectory(cdir);" << endl;
3318 out << " return kFALSE;" << endl;
3319 out << " }" << endl;
3320 out << " } else {" << endl;
3321 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3322 out << " gSystem->ChangeDirectory(cdir);" << endl;
3323 out << " return kFALSE;" << endl;
3324 out << " }" << endl;
3325 out << " // Check for SETUP.C and execute" << endl;
3326 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3327 out << " printf(\"*******************************\\n\");" << endl;
3328 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3329 out << " printf(\"*******************************\\n\");" << endl;
3330 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3331 out << " } else {" << endl;
3332 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3333 out << " gSystem->ChangeDirectory(cdir);" << endl;
3334 out << " return kFALSE;" << endl;
3335 out << " }" << endl;
3336 out << " // Restore original workdir" << endl;
3337 out << " gSystem->ChangeDirectory(cdir);" << endl;
3338 out << " return kTRUE;" << endl;
3339 out << "}" << endl;
3340 }
c57f56b7 3341 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3342 }
3343 Bool_t copy = kTRUE;
d3b18c4c 3344 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3345 if (copy) {
3346 CdWork();
3347 TString workdir = gGrid->GetHomeDirectory();
3348 workdir += fGridWorkingDir;
3349 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3350 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3351 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3352 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3353 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3354 }
3355 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3356 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3357 }
3358}
3359
0f389141 3360//______________________________________________________________________________
3361void AliAnalysisAlien::WriteMergingMacro()
3362{
3363// Write a macro to merge the outputs per master job.
3364 if (!fMergeViaJDL) return;
3365 if (!fOutputFiles.Length()) {
3366 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3367 return;
3368 }
3369 TString mergingMacro = fExecutable;
3370 mergingMacro.ReplaceAll(".sh","_merge.C");
3371 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3372 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3373 ofstream out;
3374 out.open(mergingMacro.Data(), ios::out);
3375 if (!out.good()) {
3376 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3377 return;
3378 }
5b273635 3379 Bool_t hasSTEERBase = kFALSE;
3380 Bool_t hasESD = kFALSE;
3381 Bool_t hasAOD = kFALSE;
3382 Bool_t hasANALYSIS = kFALSE;
3383 Bool_t hasANALYSISalice = kFALSE;
3384 Bool_t hasCORRFW = kFALSE;
0f389141 3385 TString func = mergingMacro;
3386 TString comment;
3387 func.ReplaceAll(".C", "");
f47d5cb4 3388 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3389 out << "{" << endl;
3390 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3391 out << " TStopwatch timer;" << endl;
3392 out << " timer.Start();" << endl << endl;
b3e07543 3393 // Reset existing include path
3394 out << "// Reset existing include path and add current directory first in the search" << endl;
3395 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3396 if (!fExecutableCommand.Contains("aliroot")) {
3397 out << "// load base root libraries" << endl;
3398 out << " gSystem->Load(\"libTree\");" << endl;
3399 out << " gSystem->Load(\"libGeom\");" << endl;
3400 out << " gSystem->Load(\"libVMC\");" << endl;
3401 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3402 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3403 }
0f389141 3404 if (fAdditionalRootLibs.Length()) {
3405 // in principle libtree /lib geom libvmc etc. can go into this list, too
3406 out << "// Add aditional libraries" << endl;
3407 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3408 TIter next(list);
3409 TObjString *str;
3410 while((str=(TObjString*)next())) {
3411 if (str->GetString().Contains(".so"))
3412 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3413 }
3414 if (list) delete list;
3415 }
0f389141 3416 out << "// Load analysis framework libraries" << endl;
3417 if (!fPackages) {
7c2cd90a 3418 if (!fExecutableCommand.Contains("aliroot")) {
3419 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3420 out << " gSystem->Load(\"libESD\");" << endl;
3421 out << " gSystem->Load(\"libAOD\");" << endl;
3422 }
0f389141 3423 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3424 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3425 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3426 } else {
3427 TIter next(fPackages);
3428 TObject *obj;
3429 TString pkgname;
5b273635 3430 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3431 while ((obj=next())) {
3432 pkgname = obj->GetName();
3433 if (pkgname == "STEERBase" ||
3434 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3435 if (pkgname == "ESD" ||
3436 pkgname == "ESD.par") hasESD = kTRUE;
3437 if (pkgname == "AOD" ||
3438 pkgname == "AOD.par") hasAOD = kTRUE;
3439 if (pkgname == "ANALYSIS" ||
3440 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3441 if (pkgname == "ANALYSISalice" ||
3442 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3443 if (pkgname == "CORRFW" ||
3444 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3445 }
5b273635 3446 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3447 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3448 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3449 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3450 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3451 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3452 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3453 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3454 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3455 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3456 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3457 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3458 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3459 out << "// Compile other par packages" << endl;
3460 next.Reset();
3461 while ((obj=next())) {
3462 pkgname = obj->GetName();
3463 if (pkgname == "STEERBase" ||
3464 pkgname == "STEERBase.par" ||
3465 pkgname == "ESD" ||
3466 pkgname == "ESD.par" ||
3467 pkgname == "AOD" ||
3468 pkgname == "AOD.par" ||
3469 pkgname == "ANALYSIS" ||
3470 pkgname == "ANALYSIS.par" ||
3471 pkgname == "ANALYSISalice" ||
3472 pkgname == "ANALYSISalice.par" ||
3473 pkgname == "CORRFW" ||
3474 pkgname == "CORRFW.par") continue;
5b273635 3475 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3476 }
3477 }
b3e07543 3478 out << "// include path" << endl;
3479 // Get the include path from the interpreter and remove entries pointing to AliRoot
3480 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3481 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3482 out << " TIter nextpath(listpaths);" << endl;
3483 out << " TObjString *pname;" << endl;
3484 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3485 out << " TString current = pname->GetName();" << endl;
3486 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3487 out << " gSystem->AddIncludePath(current);" << endl;
3488 out << " }" << endl;
3489 out << " if (listpaths) delete listpaths;" << endl;
3490 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3491 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3492 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3493 if (fAdditionalLibs.Length()) {
3494 out << "// Add aditional AliRoot libraries" << endl;
3495 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3496 TIter next(list);
3497 TObjString *str;
3498 while((str=(TObjString*)next())) {
3499 if (str->GetString().Contains(".so"))
3500 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3501 }
3502 if (list) delete list;
3503 }
3504 out << endl;
3505 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3506 if (fAnalysisSource.Length()) {
3507 TObjArray *list = fAnalysisSource.Tokenize(" ");
3508 TIter next(list);
3509 TObjString *str;
3510 while((str=(TObjString*)next())) {
3511 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3512 }
3513 if (list) delete list;
3514 }
149d288c 3515 out << endl;
3516
0f389141 3517 if (fFastReadOption) {
3518 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3519 out << "// fast xrootd reading enabled" << endl;
3520 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3521 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3522 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3523 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3524 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3525 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3526 }
3527 // Change temp directory to current one
3528 out << "// Set temporary merging directory to current one" << endl;
3529 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3530 out << "// Connect to AliEn" << endl;
3531 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3532 out << " TString outputDir = dir;" << endl;
f790bc1b 3533 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3534 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3535 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3536 out << " TIter *iter = new TIter(list);" << endl;
3537 out << " TObjString *str;" << endl;
a2f5fc01 3538 out << " TString outputFile;" << endl;
0f389141 3539 out << " Bool_t merged = kTRUE;" << endl;
3540 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3541 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3542 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3543 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3544 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3545 out << " // Skip already merged outputs" << endl;
a2f5fc01 3546 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3547 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3548 out << " continue;" << endl;
3549 out << " }" << endl;
a2f5fc01 3550 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3551 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3552 out << " if (!merged) {" << endl;
a2f5fc01 3553 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3554 out << " return;" << endl;
0f389141 3555 out << " }" << endl;
3556 out << " }" << endl;
7c2cd90a 3557 out << " // all outputs merged, validate" << endl;
3558 out << " ofstream out;" << endl;
3559 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3560 out << " out.close();" << endl;
3561 out << " // read the analysis manager from file" << endl;
f866cba5 3562 TString analysisFile = fExecutable;
3563 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3564 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3565 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3566 out << " if (!file) return;" << endl;
f866cba5 3567 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3568 out << " AliAnalysisManager *mgr = 0;" << endl;
3569 out << " TKey *key;" << endl;
3570 out << " while ((key=(TKey*)nextkey())) {" << endl;
3571 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3572 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3573 out << " };" << endl;
3574 out << " if (!mgr) {" << endl;
3575 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3576 out << " return;" << endl;
3577 out << " }" << endl << endl;
5c4250fc 3578 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3579 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3580 out << " mgr->PrintStatus();" << endl;
3581 if (AliAnalysisManager::GetAnalysisManager()) {
3582 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3583 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3584 } else {
9a8288ff 3585 if (TestBit(AliAnalysisGrid::kTest))
3586 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3587 else
3588 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3589 }
3590 }
9a8288ff 3591 out << " TTree *tree = NULL;" << endl;
3592 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3593 out << "}" << endl << endl;
5b273635 3594 if (hasANALYSISalice) {
3595 out <<"//________________________________________________________________________________" << endl;
3596 out << "Bool_t SetupPar(const char *package) {" << endl;
3597 out << "// Compile the package and set it up." << endl;
3598 out << " TString pkgdir = package;" << endl;
3599 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3600 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3601 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3602 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3603 out << " // Check for BUILD.sh and execute" << endl;
3604 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3605 out << " printf(\"*******************************\\n\");" << endl;
3606 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3607 out << " printf(\"*******************************\\n\");" << endl;
3608 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3609 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3610 out << " gSystem->ChangeDirectory(cdir);" << endl;
3611 out << " return kFALSE;" << endl;
3612 out << " }" << endl;
3613 out << " } else {" << endl;
3614 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3615 out << " gSystem->ChangeDirectory(cdir);" << endl;
3616 out << " return kFALSE;" << endl;
3617 out << " }" << endl;
3618 out << " // Check for SETUP.C and execute" << endl;
3619 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3620 out << " printf(\"*******************************\\n\");" << endl;
3621 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3622 out << " printf(\"*******************************\\n\");" << endl;
3623 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3624 out << " } else {" << endl;
3625 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3626 out << " gSystem->ChangeDirectory(cdir);" << endl;
3627 out << " return kFALSE;" << endl;
3628 out << " }" << endl;
3629 out << " // Restore original workdir" << endl;
3630 out << " gSystem->ChangeDirectory(cdir);" << endl;
3631 out << " return kTRUE;" << endl;
3632 out << "}" << endl;
3633 }
0f389141 3634 }
3635 Bool_t copy = kTRUE;
d3b18c4c 3636 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3637 if (copy) {
3638 CdWork();
3639 TString workdir = gGrid->GetHomeDirectory();
3640 workdir += fGridWorkingDir;
3641 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3642 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3643 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3644 }
3645}
3646
3647//______________________________________________________________________________
3648Bool_t AliAnalysisAlien::SetupPar(const char *package)
3649{
205b201f 3650// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3651// Note that for loading the compiled library. The current directory should have precedence in
3652// LD_LIBRARY_PATH
3653 TString pkgdir = package;
3654 pkgdir.ReplaceAll(".par","");
b50a41c3 3655 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3656 TString cdir = gSystem->WorkingDirectory();
3657 gSystem->ChangeDirectory(pkgdir);
3658 // Check for BUILD.sh and execute
3659 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3660 printf("**************************************************\n");
3661 printf("*** Building PAR archive %s\n", package);
3662 printf("**************************************************\n");
3663 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3664 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3665 gSystem->ChangeDirectory(cdir);
3666 return kFALSE;
3667 }
3668 } else {
3669 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3670 gSystem->ChangeDirectory(cdir);
3671 return kFALSE;
3672 }
3673 // Check for SETUP.C and execute
3674 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3675 printf("**************************************************\n");
3676 printf("*** Setup PAR archive %s\n", package);
3677 printf("**************************************************\n");
3678 gROOT->Macro("PROOF-INF/SETUP.C");
3679 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3680 } else {
3681 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3682 gSystem->ChangeDirectory(cdir);
3683 return kFALSE;
3684 }
3685 // Restore original workdir
3686 gSystem->ChangeDirectory(cdir);
3687 return kTRUE;
3688}
3689
c57f56b7 3690//______________________________________________________________________________
3691void AliAnalysisAlien::WriteExecutable()
3692{
3693// Generate the alien executable script.
3694 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3695 ofstream out;
3696 out.open(fExecutable.Data(), ios::out);
3697 if (out.bad()) {
5513444a 3698 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3699 return;
3700 }
3701 out << "#!/bin/bash" << endl;
2a4d5166 3702 // Make sure we can properly compile par files
3703 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3704 out << "echo \"=========================================\"" << endl;
3705 out << "echo \"############## PATH : ##############\"" << endl;
3706 out << "echo $PATH" << endl;
3707 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3708 out << "echo $LD_LIBRARY_PATH" << endl;
3709 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3710 out << "echo $ROOTSYS" << endl;
3711 out << "echo \"############## which root : ##############\"" << endl;
3712 out << "which root" << endl;
3713 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3714 out << "echo $ALICE_ROOT" << endl;
3715 out << "echo \"############## which aliroot : ##############\"" << endl;
3716 out << "which aliroot" << endl;
9c5ddadc 3717 out << "echo \"############## system limits : ##############\"" << endl;
3718 out << "ulimit -a" << endl;
3719 out << "echo \"############## memory : ##############\"" << endl;
3720 out << "free -m" << endl;
c57f56b7 3721 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3722 out << fExecutableCommand << " ";
631c0b05 3723 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3724 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3725 out << "echo \"############## memory after: ##############\"" << endl;
3726 out << "free -m" << endl;
c57f56b7 3727 }
3728 Bool_t copy = kTRUE;
d3b18c4c 3729 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3730 if (copy) {
3731 CdWork();
3732 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3733 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3734 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3735 workdir += fGridWorkingDir;
3736 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3737 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3738 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3739 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3740 }
3741}
3742
0f389141 3743//______________________________________________________________________________
3744void AliAnalysisAlien::WriteMergeExecutable()
3745{
3746// Generate the alien executable script for the merging job.
3747 if (!fMergeViaJDL) return;
3748 TString mergeExec = fExecutable;
3749 mergeExec.ReplaceAll(".sh", "_merge.sh");
3750 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3751 ofstream out;
3752 out.open(mergeExec.Data(), ios::out);
3753 if (out.bad()) {
3754 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3755 return;
3756 }
3757 out << "#!/bin/bash" << endl;
2a4d5166 3758 // Make sure we can properly compile par files
3759 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3760 out << "echo \"=========================================\"" << endl;
3761 out << "echo \"############## PATH : ##############\"" << endl;
3762 out << "echo $PATH" << endl;
3763 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3764 out << "echo $LD_LIBRARY_PATH" << endl;
3765 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3766 out << "echo $ROOTSYS" << endl;
3767 out << "echo \"############## which root : ##############\"" << endl;
3768 out << "which root" << endl;
3769 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3770 out << "echo $ALICE_ROOT" << endl;
3771 out << "echo \"############## which aliroot : ##############\"" << endl;
3772 out << "which aliroot" << endl;
3773 out << "echo \"############## system limits : ##############\"" << endl;
3774 out << "ulimit -a" << endl;
3775 out << "echo \"############## memory : ##############\"" << endl;
3776 out << "free -m" << endl;
3777 out << "echo \"=========================================\"" << endl << endl;
0f389141 3778 TString mergeMacro = fExecutable;
3779 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3780 if (IsOneStageMerging())
3781 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3782 else
f47d5cb4 3783 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3784 out << fExecutableCommand << " " << "$ARG" << endl;
3785 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3786 out << "echo \"############## memory after: ##############\"" << endl;
3787 out << "free -m" << endl;
0f389141 3788 }
3789 Bool_t copy = kTRUE;
d3b18c4c 3790 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3791 if (copy) {
3792 CdWork();
3793 TString workdir = gGrid->GetHomeDirectory();
3794 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3795 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3796 workdir += fGridWorkingDir;
3797 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3798 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3799 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3800 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3801 }
3802}
3803
c57f56b7 3804//______________________________________________________________________________
5513444a 3805void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3806{
3807// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3808// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3809// Next lines: full_path_to_dataset XXX (XXX is a string)
3810// To submit, one has to: submit jdl XXX for all lines
3811 ofstream out;
3812 out.open(filename, ios::out);
3813 if (out.bad()) {
3814 Error("WriteProductionFile", "Bad file name: %s", filename);
3815 return;
3816 }
d3b18c4c 3817 TString workdir;
3818 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3819 workdir = gGrid->GetHomeDirectory();
5513444a 3820 workdir += fGridWorkingDir;
f5e8c702 3821 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3822 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3823 out << locjdl << " " << njobspermaster << endl;
5513444a 3824 Int_t nmasterjobs = fInputFiles->GetEntries();
3825 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3826 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3827 runOutDir.ReplaceAll(".xml", "");
3828 if (fOutputToRunNo)
3829 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3830 else
3831 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3832 }
d3b18c4c 3833 if (gGrid) {
3834 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3835 if (FileExists(filename)) gGrid->Rm(filename);
3836 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3837 }
5513444a 3838}
3839
3840//______________________________________________________________________________
0f389141 3841void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3842{
3843// Generate the alien validation script.
3844 // Generate the validation script
3845 TObjString *os;
d3b18c4c 3846 if (fValidationScript.IsNull()) {
3847 fValidationScript = fExecutable;
3848 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3849 }
3850 TString validationScript = fValidationScript;
3851 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3852 if (!Connect()) {
3853 Error("WriteValidationScript", "Alien connection required");
3854 return;
3855 }
d3b18c4c 3856 if (!fTerminateFiles.IsNull()) {
3857 fTerminateFiles.Strip();
3858 fTerminateFiles.ReplaceAll(" ",",");
3859 }
a2f5fc01 3860 TString outStream = "";
3861 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3862 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3863 ofstream out;
0d5d317c 3864 out.open(validationScript, ios::out);
c57f56b7 3865 out << "#!/bin/bash" << endl;
3866 out << "##################################################" << endl;
3867 out << "validateout=`dirname $0`" << endl;
3868 out << "validatetime=`date`" << endl;
3869 out << "validated=\"0\";" << endl;
3870 out << "error=0" << endl;
3871 out << "if [ -z $validateout ]" << endl;
3872 out << "then" << endl;
3873 out << " validateout=\".\"" << endl;
3874 out << "fi" << endl << endl;
3875 out << "cd $validateout;" << endl;
3876 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3877 out << "echo \"*******************************************************\"" << outStream << endl;
3878 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3879 out << "" << endl;
a2f5fc01 3880 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3881 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3882 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3883 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3884 out << "ls -la ./" << outStream << endl;
3885 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3886 out << "##################################################" << endl;
ebec370a 3887 out << "" << endl;
3888
3889 out << "if [ ! -f stderr ] ; then" << endl;
3890 out << " error=1" << endl;
a2f5fc01 3891 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3892 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3893 out << "fi" << endl;
3894
b34c9f51 3895 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3896 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3897 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3898 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3899 out << "" << endl;
3900
ebec370a 3901 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3902 out << " error=1" << endl;
a2f5fc01 3903 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3904 out << " echo \"$parArch\" " << outStream << endl;
3905 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3906 out << "fi" << endl;
3907
3908 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3909 out << " error=1" << endl;
a2f5fc01 3910 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3911 out << " echo \"$segViol\" " << outStream << endl;
3912 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3913 out << "fi" << endl;
3914
3915 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3916 out << " error=1" << endl;
a2f5fc01 3917 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3918 out << " echo \"$segFault\" " << outStream << endl;
3919 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3920 out << "fi" << endl;
3921
b34c9f51 3922 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3923 out << " error=1" << endl;
a2f5fc01 3924 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3925 out << " echo \"$glibcErr\" " << outStream << endl;
3926 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3927 out << "fi" << endl;
3928
ebec370a 3929 // Part dedicated to the specific analyses running into the train
3930
d3b18c4c 3931 TString outputFiles = fOutputFiles;
3932 if (merge && !fTerminateFiles.IsNull()) {
3933 outputFiles += ",";
3934 outputFiles += fTerminateFiles;
3935 }
3936 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3937 TIter next1(arr);
a2f5fc01 3938 TString outputFile;
bbd88db8 3939 while (!merge && (os=(TObjString*)next1())) {
3940 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3941 outputFile = os->GetString();
3942 Int_t index = outputFile.Index("@");
3943 if (index > 0) outputFile.Remove(index);
bbd88db8 3944 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3945 if (outputFile.Contains("*")) continue;
3946 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3947 out << " error=1" << endl;
7c2cd90a 3948 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3949 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3950 out << "fi" << endl;
3951 }
3952 delete arr;
7c2cd90a 3953 out << "if ! [ -f outputs_valid ] ; then" << endl;
3954 out << " error=1" << endl;
3955 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3956 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3957 out << "fi" << endl;
923e2ca5 3958
c57f56b7 3959 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3960 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3961 if (!IsKeepLogs()) {
3962 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3963 outStream = "";
149d288c 3964 out << " rm -f std*" << endl;
3965 }
c57f56b7 3966 out << "fi" << endl;
3967
a2f5fc01 3968 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3969 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3970 out << "cd -" << endl;
3971 out << "exit $error" << endl;
3972 }
3973 Bool_t copy = kTRUE;
d3b18c4c 3974 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3975 if (copy) {
3976 CdWork();
3977 TString workdir = gGrid->GetHomeDirectory();
3978 workdir += fGridWorkingDir;
d3b18c4c 3979 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 3980 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3981 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3982 }
3983}