]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fixing signed/unsigned comparison warning.
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
b3e07543 61 fMaxMergeStages(0),
d2a409b2 62 fNsubmitted(0),
a3e84053 63 fProductionMode(0),
cd11251e 64 fOutputToRunNo(0),
0f389141 65 fMergeViaJDL(0),
66 fFastReadOption(0),
e1c22e21 67 fOverwriteMode(1),
149d288c 68 fNreplicas(2),
3bdcb562 69 fNproofWorkers(0),
27734f0e 70 fNproofWorkersPerSlave(0),
3bdcb562 71 fProofReset(0),
c57f56b7 72 fRunNumbers(),
73 fExecutable(),
0a1c1f7f 74 fExecutableCommand(),
c57f56b7 75 fArguments(),
631c0b05 76 fExecutableArgs(),
c57f56b7 77 fAnalysisMacro(),
78 fAnalysisSource(),
d3b18c4c 79 fValidationScript(),
d5c6455a 80 fAdditionalRootLibs(),
c57f56b7 81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
648174cf 86 fExternalPackages(),
c57f56b7 87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
e7c71df0 95 fDatasetName(),
c57f56b7 96 fJDLName(),
d3b18c4c 97 fTerminateFiles(),
bb885a9e 98 fMergeExcludes(),
f965131e 99 fIncludePath(),
bb885a9e 100 fCloseSE(),
0df6ccf2 101 fFriendChainName(),
c6cb3634 102 fJobTag(),
648174cf 103 fOutputSingle(),
5fce53f4 104 fRunPrefix(),
3bdcb562 105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
f47d5cb4 110 fMergeDirName(),
4e5c5506 111 fInputFiles(0),
112 fPackages(0)
c57f56b7 113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
0f389141 122 fMergingJDL(NULL),
c57f56b7 123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
bb885a9e 128 fNtestFiles(0),
319593fb 129 fNrunsPerMaster(0),
16a4353c 130 fMaxMergeFiles(0),
b3e07543 131 fMaxMergeStages(0),
d2a409b2 132 fNsubmitted(0),
a3e84053 133 fProductionMode(0),
cd11251e 134 fOutputToRunNo(0),
0f389141 135 fMergeViaJDL(0),
136 fFastReadOption(0),
e1c22e21 137 fOverwriteMode(1),
149d288c 138 fNreplicas(2),
3bdcb562 139 fNproofWorkers(0),
27734f0e 140 fNproofWorkersPerSlave(0),
3bdcb562 141 fProofReset(0),
c57f56b7 142 fRunNumbers(),
143 fExecutable(),
0a1c1f7f 144 fExecutableCommand(),
c57f56b7 145 fArguments(),
631c0b05 146 fExecutableArgs(),
c57f56b7 147 fAnalysisMacro(),
148 fAnalysisSource(),
d3b18c4c 149 fValidationScript(),
d5c6455a 150 fAdditionalRootLibs(),
c57f56b7 151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
648174cf 156 fExternalPackages(),
c57f56b7 157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
e7c71df0 165 fDatasetName(),
c57f56b7 166 fJDLName(),
d3b18c4c 167 fTerminateFiles(),
bb885a9e 168 fMergeExcludes(),
f965131e 169 fIncludePath(),
bb885a9e 170 fCloseSE(),
0df6ccf2 171 fFriendChainName(),
c6cb3634 172 fJobTag(),
648174cf 173 fOutputSingle(),
5fce53f4 174 fRunPrefix(),
3bdcb562 175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
f47d5cb4 180 fMergeDirName(),
4e5c5506 181 fInputFiles(0),
182 fPackages(0)
c57f56b7 183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
0f389141 192 fMergingJDL(NULL),
c57f56b7 193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 198 fNtestFiles(other.fNtestFiles),
319593fb 199 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 200 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 201 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 202 fNsubmitted(other.fNsubmitted),
a3e84053 203 fProductionMode(other.fProductionMode),
cd11251e 204 fOutputToRunNo(other.fOutputToRunNo),
0f389141 205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
149d288c 208 fNreplicas(other.fNreplicas),
3bdcb562 209 fNproofWorkers(other.fNproofWorkers),
27734f0e 210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 211 fProofReset(other.fProofReset),
c57f56b7 212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
0a1c1f7f 214 fExecutableCommand(other.fExecutableCommand),
c57f56b7 215 fArguments(other.fArguments),
631c0b05 216 fExecutableArgs(other.fExecutableArgs),
c57f56b7 217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 219 fValidationScript(other.fValidationScript),
d5c6455a 220 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
648174cf 226 fExternalPackages(other.fExternalPackages),
c57f56b7 227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
e7c71df0 235 fDatasetName(other.fDatasetName),
c57f56b7 236 fJDLName(other.fJDLName),
d3b18c4c 237 fTerminateFiles(other.fTerminateFiles),
bb885a9e 238 fMergeExcludes(other.fMergeExcludes),
f965131e 239 fIncludePath(other.fIncludePath),
bb885a9e 240 fCloseSE(other.fCloseSE),
0df6ccf2 241 fFriendChainName(other.fFriendChainName),
c6cb3634 242 fJobTag(other.fJobTag),
648174cf 243 fOutputSingle(other.fOutputSingle),
5fce53f4 244 fRunPrefix(other.fRunPrefix),
3bdcb562 245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
f47d5cb4 250 fMergeDirName(other.fMergeDirName),
4e5c5506 251 fInputFiles(0),
252 fPackages(0)
c57f56b7 253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
c57f56b7 259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
4e5c5506 266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
c57f56b7 273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
0f389141 280 if (fMergingJDL) delete fMergingJDL;
c57f56b7 281 if (fInputFiles) delete fInputFiles;
4e5c5506 282 if (fPackages) delete fPackages;
c57f56b7 283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 298 fNtestFiles = other.fNtestFiles;
a3e84053 299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 301 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
cd11251e 304 fOutputToRunNo = other.fOutputToRunNo;
0f389141 305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
149d288c 308 fNreplicas = other.fNreplicas;
3bdcb562 309 fNproofWorkers = other.fNproofWorkers;
27734f0e 310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 311 fProofReset = other.fProofReset;
c57f56b7 312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
0a1c1f7f 314 fExecutableCommand = other.fExecutableCommand;
c57f56b7 315 fArguments = other.fArguments;
631c0b05 316 fExecutableArgs = other.fExecutableArgs;
c57f56b7 317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 319 fValidationScript = other.fValidationScript;
d5c6455a 320 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
648174cf 326 fExternalPackages = other.fExternalPackages;
c57f56b7 327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
e7c71df0 335 fDatasetName = other.fDatasetName;
c57f56b7 336 fJDLName = other.fJDLName;
d3b18c4c 337 fTerminateFiles = other.fTerminateFiles;
bb885a9e 338 fMergeExcludes = other.fMergeExcludes;
f965131e 339 fIncludePath = other.fIncludePath;
bb885a9e 340 fCloseSE = other.fCloseSE;
0df6ccf2 341 fFriendChainName = other.fFriendChainName;
c6cb3634 342 fJobTag = other.fJobTag;
648174cf 343 fOutputSingle = other.fOutputSingle;
5fce53f4 344 fRunPrefix = other.fRunPrefix;
3bdcb562 345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
f47d5cb4 350 fMergeDirName = other.fMergeDirName;
c57f56b7 351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
4e5c5506 358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
c57f56b7 365 }
366 return *this;
367}
368
74013535 369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
f965131e 377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 391 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 392}
393
98690895 394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunList(const char* runList)
396{
397// Add several runs into the list of runs; they are expected to be separated by a blank character.
398 TString sList = runList;
399 TObjArray *list = sList.Tokenize(" ");
400 Int_t n = list->GetEntries();
401 for (Int_t i = 0; i < n; i++) {
402 TObjString *os = (TObjString*)list->At(i);
403 AddRunNumber(os->GetString().Atoi());
404 }
405 delete list;
406}
407
ee75cfc3 408//______________________________________________________________________________
409void AliAnalysisAlien::AddRunNumber(const char* run)
410{
411// Add a run number to the list of runs to be processed.
412 if (fRunNumbers.Length()) fRunNumbers += " ";
413 fRunNumbers += run;
414}
415
c57f56b7 416//______________________________________________________________________________
417void AliAnalysisAlien::AddDataFile(const char *lfn)
418{
419// Adds a data file to the input to be analysed. The file should be a valid LFN
420// or point to an existing file in the alien workdir.
421 if (!fInputFiles) fInputFiles = new TObjArray();
422 fInputFiles->Add(new TObjString(lfn));
423}
648174cf 424
425//______________________________________________________________________________
426void AliAnalysisAlien::AddExternalPackage(const char *package)
427{
428// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
429 if (fExternalPackages) fExternalPackages += " ";
430 fExternalPackages += package;
431}
432
c57f56b7 433//______________________________________________________________________________
434Bool_t AliAnalysisAlien::Connect()
435{
436// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
437 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 438 if (fProductionMode) return kTRUE;
c57f56b7 439 if (!gGrid) {
440 Info("Connect", "Trying to connect to AliEn ...");
441 TGrid::Connect("alien://");
442 }
443 if (!gGrid || !gGrid->IsConnected()) {
444 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
445 return kFALSE;
446 }
447 fUser = gGrid->GetUser();
448 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
449 return kTRUE;
450}
451
452//______________________________________________________________________________
453void AliAnalysisAlien::CdWork()
454{
455// Check validity of alien workspace. Create directory if possible.
456 if (!Connect()) {
457 Error("CdWork", "Alien connection required");
458 return;
459 }
460 TString homedir = gGrid->GetHomeDirectory();
461 TString workdir = homedir + fGridWorkingDir;
923e2ca5 462 if (DirectoryExists(workdir)) {
463 gGrid->Cd(workdir);
464 return;
465 }
466 // Work directory not existing - create it
467 gGrid->Cd(homedir);
b93f8109 468 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 469 gGrid->Cd(fGridWorkingDir);
d3b18c4c 470 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 471 } else {
d3b18c4c 472 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 473 workdir.Data(), homedir.Data());
474 fGridWorkingDir = "";
475 }
c57f56b7 476}
477
348be253 478//______________________________________________________________________________
479Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
480{
481// Check if file copying is possible.
d3b18c4c 482 if (fProductionMode) return kTRUE;
348be253 483 if (!Connect()) {
484 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
485 return kFALSE;
486 }
d3339be3 487 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
488 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 489 // Check if alien_CLOSE_SE is defined
490 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
491 if (!closeSE.IsNull()) {
492 Info("CheckFileCopy", "Your current close storage is pointing to: \
493 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
494 } else {
495 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
496 }
497 // Check if grid directory exists.
498 if (!DirectoryExists(alienpath)) {
499 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
500 return kFALSE;
501 }
d3339be3 502 TFile f("plugin_test_copy", "RECREATE");
348be253 503 // User may not have write permissions to current directory
504 if (f.IsZombie()) {
505 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
506 gSystem->WorkingDirectory());
507 return kFALSE;
508 }
509 f.Close();
d3339be3 510 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
511 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
512 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 513 \n# 1. Make sure you have write permissions there. If this is the case: \
514 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
515 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
516 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
517 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
518 gSystem->Unlink(f.GetName());
519 return kFALSE;
520 }
521 gSystem->Unlink(f.GetName());
d3339be3 522 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
523 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 524 return kTRUE;
525}
526
c57f56b7 527//______________________________________________________________________________
528Bool_t AliAnalysisAlien::CheckInputData()
529{
530// Check validity of input data. If necessary, create xml files.
d3b18c4c 531 if (fProductionMode) return kTRUE;
a8739e8a 532 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 533 if (!fGridDataDir.Length()) {
534 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
535 return kFALSE;
536 }
9f020952 537 if (fMergeViaJDL) {
538 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
539 return kFALSE;
540 }
d2a409b2 541 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 542 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
543 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 544 return kTRUE;
c57f56b7 545 }
546 // Process declared files
a2f5fc01 547 Bool_t isCollection = kFALSE;
548 Bool_t isXml = kFALSE;
549 Bool_t useTags = kFALSE;
c57f56b7 550 Bool_t checked = kFALSE;
d3b18c4c 551 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 552 TString file;
553 TString workdir = gGrid->GetHomeDirectory();
554 workdir += fGridWorkingDir;
555 if (fInputFiles) {
556 TObjString *objstr;
557 TIter next(fInputFiles);
558 while ((objstr=(TObjString*)next())) {
559 file = workdir;
560 file += "/";
561 file += objstr->GetString();
562 // Store full lfn path
563 if (FileExists(file)) objstr->SetString(file);
564 else {
565 file = objstr->GetName();
566 if (!FileExists(objstr->GetName())) {
567 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
568 objstr->GetName(), workdir.Data());
569 return kFALSE;
570 }
571 }
572 Bool_t iscoll, isxml, usetags;
573 CheckDataType(file, iscoll, isxml, usetags);
574 if (!checked) {
575 checked = kTRUE;
a2f5fc01 576 isCollection = iscoll;
577 isXml = isxml;
578 useTags = usetags;
579 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 580 } else {
a2f5fc01 581 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 582 Error("CheckInputData", "Some conflict was found in the types of inputs");
583 return kFALSE;
584 }
585 }
586 }
587 }
588 // Process requested run numbers
a8739e8a 589 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 590 // Check validity of alien data directory
591 if (!fGridDataDir.Length()) {
592 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
593 return kFALSE;
594 }
923e2ca5 595 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 596 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
597 return kFALSE;
598 }
a2f5fc01 599 if (isCollection) {
c57f56b7 600 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
601 return kFALSE;
602 }
603
a2f5fc01 604 if (checked && !isXml) {
c57f56b7 605 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
606 return kFALSE;
607 }
608 // Check validity of run number(s)
609 TObjArray *arr;
610 TObjString *os;
74013535 611 TString format;
319593fb 612 Int_t nruns = 0;
904f9f5f 613 TString schunk, schunk2;
c57f56b7 614 TString path;
615 if (!checked) {
616 checked = kTRUE;
a2f5fc01 617 useTags = fDataPattern.Contains("tag");
618 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 619 }
a2f5fc01 620 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 621 Error("CheckInputData", "Cannot mix input files using/not using tags");
622 return kFALSE;
623 }
624 if (fRunNumbers.Length()) {
a8739e8a 625 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 626 arr = fRunNumbers.Tokenize(" ");
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
629 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 630 if (!DirectoryExists(path)) {
631 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 632 continue;
c57f56b7 633 }
634 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
635 TString msg = "\n##### file: ";
636 msg += path;
637 msg += " type: xml_collection;";
a2f5fc01 638 if (useTags) msg += " using_tags: Yes";
c57f56b7 639 else msg += " using_tags: No";
23329835 640 Info("CheckDataType", "%s", msg.Data());
319593fb 641 if (fNrunsPerMaster<2) {
d2a409b2 642 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 643 } else {
644 nruns++;
645 if (((nruns-1)%fNrunsPerMaster) == 0) {
646 schunk = os->GetString();
647 }
648 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
649 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 650 AddDataFile(schunk);
319593fb 651 }
c57f56b7 652 }
653 delete arr;
a8739e8a 654 } else {
655 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
656 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 657 format = Form("%%s/%s ", fRunPrefix.Data());
658 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 659 if (!DirectoryExists(path)) {
a8739e8a 660 continue;
661 }
74013535 662 format = Form("%%s/%s.xml", fRunPrefix.Data());
663 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 664 TString msg = "\n##### file: ";
665 msg += path;
666 msg += " type: xml_collection;";
a2f5fc01 667 if (useTags) msg += " using_tags: Yes";
a8739e8a 668 else msg += " using_tags: No";
23329835 669 Info("CheckDataType", "%s", msg.Data());
319593fb 670 if (fNrunsPerMaster<2) {
74013535 671 format = Form("%s.xml", fRunPrefix.Data());
672 AddDataFile(Form(format.Data(),irun));
319593fb 673 } else {
674 nruns++;
675 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 676 schunk = Form(fRunPrefix.Data(),irun);
319593fb 677 }
74013535 678 format = Form("_%s.xml", fRunPrefix.Data());
679 schunk2 = Form(format.Data(), irun);
319593fb 680 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 681 schunk += schunk2;
d2a409b2 682 AddDataFile(schunk);
319593fb 683 }
a8739e8a 684 }
904f9f5f 685 if (!fInputFiles) {
686 schunk += schunk2;
687 AddDataFile(schunk);
688 }
c57f56b7 689 }
690 return kTRUE;
691}
692
693//______________________________________________________________________________
694Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
695{
696// Create dataset for the grid data directory + run number.
d3b18c4c 697 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 698 if (!Connect()) {
699 Error("CreateDataset", "Cannot create dataset with no grid connection");
700 return kFALSE;
701 }
702
703 // Cd workspace
d3b18c4c 704 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 705 TString workdir = gGrid->GetHomeDirectory();
706 workdir += fGridWorkingDir;
707
708 // Compose the 'find' command arguments
74013535 709 TString format;
c57f56b7 710 TString command;
711 TString options = "-x collection ";
bb885a9e 712 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 713 TString conditions = "";
714
715 TString file;
716 TString path;
319593fb 717 Int_t nruns = 0;
904f9f5f 718 TString schunk, schunk2;
ab254fd1 719 TGridCollection *cbase=0, *cadd=0;
d2a409b2 720 if (!fRunNumbers.Length() && !fRunRange[0]) {
721 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
722 // Make a single data collection from data directory.
723 path = fGridDataDir;
923e2ca5 724 if (!DirectoryExists(path)) {
d2a409b2 725 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
726 return kFALSE;
727 }
923e2ca5 728// CdWork();
d2a409b2 729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 731 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 732 command = "find ";
733 command += options;
734 command += path;
735 command += " ";
736 command += pattern;
737 command += conditions;
84fcd93f 738 printf("command: %s\n", command.Data());
d2a409b2 739 TGridResult *res = gGrid->Command(command);
740 if (res) delete res;
741 // Write standard output to file
742 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 743 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
744 Bool_t nullFile = kFALSE;
745 if (!hasGrep) {
d3339be3 746 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
747 } else {
a2f5fc01 748 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
749 if (nullFile) {
defd7a3a 750 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
751 return kFALSE;
752 }
d3339be3 753 }
0f389141 754 }
755 Bool_t fileExists = FileExists(file);
756 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 757 // Copy xml file to alien space
0f389141 758 if (fileExists) gGrid->Rm(file);
d2a409b2 759 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
760 if (!FileExists(file)) {
761 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
762 return kFALSE;
763 }
764 // Update list of files to be processed.
765 }
766 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
767 return kTRUE;
768 }
c57f56b7 769 // Several runs
a2f5fc01 770 Bool_t nullResult = kTRUE;
a8739e8a 771 if (fRunNumbers.Length()) {
772 TObjArray *arr = fRunNumbers.Tokenize(" ");
773 TObjString *os;
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 777 if (!DirectoryExists(path)) continue;
778// CdWork();
a8739e8a 779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
780 else file = Form("%s.xml", os->GetString().Data());
319593fb 781 // If local collection file does not exist, create it via 'find' command.
d3339be3 782 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 783 command = "find ";
784 command += options;
785 command += path;
786 command += pattern;
787 command += conditions;
788 TGridResult *res = gGrid->Command(command);
789 if (res) delete res;
790 // Write standard output to file
791 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 792 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
793 Bool_t nullFile = kFALSE;
794 if (!hasGrep) {
d3339be3 795 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
796 } else {
a2f5fc01 797 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
798 if (nullFile) {
defd7a3a 799 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
800 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
801 continue;
802 }
d3339be3 803 }
a2f5fc01 804 nullResult = kFALSE;
d3339be3 805 }
a8739e8a 806 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 807 // Check if there is one run per master job.
808 if (fNrunsPerMaster<2) {
809 if (FileExists(file)) {
0f389141 810 if (fOverwriteMode) gGrid->Rm(file);
811 else {
812 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
813 continue;
814 }
319593fb 815 }
816 // Copy xml file to alien space
817 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
818 if (!FileExists(file)) {
819 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
820 delete arr;
821 return kFALSE;
822 }
823 } else {
824 nruns++;
825 if (((nruns-1)%fNrunsPerMaster) == 0) {
826 schunk = os->GetString();
827 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
828 } else {
829 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 830 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 831 cbase->Add(cadd);
832 delete cadd;
833 }
834 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
835 continue;
836 }
837 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 838 if (FileExists(schunk)) {
839 if (fOverwriteMode) gGrid->Rm(file);
840 else {
841 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
842 continue;
843 }
319593fb 844 }
84fcd93f 845 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 846 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 847 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 848 if (!FileExists(schunk)) {
849 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
850 delete arr;
851 return kFALSE;
852 }
d3339be3 853 }
a8739e8a 854 }
855 delete arr;
a2f5fc01 856 if (nullResult) {
d3339be3 857 Error("CreateDataset", "No valid dataset corresponding to the query!");
858 return kFALSE;
859 }
a8739e8a 860 } else {
861 // Process a full run range.
862 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 863 format = Form("%%s/%s ", fRunPrefix.Data());
864 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 865 if (!DirectoryExists(path)) continue;
866// CdWork();
74013535 867 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 868 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 869 else file = Form(format.Data(), irun);
0f389141 870 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
871 if (fOverwriteMode) gGrid->Rm(file);
872 else {
873 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
874 continue;
875 }
a8739e8a 876 }
319593fb 877 // If local collection file does not exist, create it via 'find' command.
d3339be3 878 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 879 command = "find ";
880 command += options;
881 command += path;
882 command += pattern;
883 command += conditions;
884 TGridResult *res = gGrid->Command(command);
885 if (res) delete res;
886 // Write standard output to file
887 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 888 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
889 Bool_t nullFile = kFALSE;
890 if (!hasGrep) {
d3339be3 891 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
892 } else {
a2f5fc01 893 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
894 if (nullFile) {
defd7a3a 895 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
896 continue;
897 }
d3339be3 898 }
a2f5fc01 899 nullResult = kFALSE;
319593fb 900 }
a8739e8a 901 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 902 // Check if there is one run per master job.
903 if (fNrunsPerMaster<2) {
904 if (FileExists(file)) {
0f389141 905 if (fOverwriteMode) gGrid->Rm(file);
906 else {
907 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
908 continue;
909 }
319593fb 910 }
911 // Copy xml file to alien space
912 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
913 if (!FileExists(file)) {
914 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
915 return kFALSE;
916 }
917 } else {
918 nruns++;
95e5b448 919 // Check if the collection for the chunk exist locally.
920 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 921 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
922 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
923 else continue;
924 }
84fcd93f 925 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 926 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 927 schunk = Form(fRunPrefix.Data(), irun);
319593fb 928 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
929 } else {
930 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 cbase->Add(cadd);
932 delete cadd;
933 }
74013535 934 format = Form("%%s_%s.xml", fRunPrefix.Data());
935 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 936 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 937 continue;
938 }
904f9f5f 939 schunk = schunk2;
319593fb 940 if (FileExists(schunk)) {
0f389141 941 if (fOverwriteMode) gGrid->Rm(schunk);
942 else {
943 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
944 continue;
945 }
319593fb 946 }
84fcd93f 947 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 948 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 949 if (FileExists(schunk)) {
0f389141 950 if (fOverwriteMode) gGrid->Rm(schunk);
951 else {
952 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
953 continue;
954 }
95e5b448 955 }
319593fb 956 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
957 if (!FileExists(schunk)) {
958 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
959 return kFALSE;
960 }
961 }
c57f56b7 962 }
a2f5fc01 963 if (nullResult) {
d3339be3 964 Error("CreateDataset", "No valid dataset corresponding to the query!");
965 return kFALSE;
966 }
a8739e8a 967 }
c57f56b7 968 return kTRUE;
969}
970
971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CreateJDL()
973{
974// Generate a JDL file according to current settings. The name of the file is
975// specified by fJDLName.
976 Bool_t error = kFALSE;
977 TObjArray *arr = 0;
978 Bool_t copy = kTRUE;
d3b18c4c 979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 980 Bool_t generate = kTRUE;
981 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
982 if (!Connect()) {
983 Error("CreateJDL", "Alien connection required");
984 return kFALSE;
985 }
986 // Check validity of alien workspace
d3b18c4c 987 TString workdir;
988 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
989 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 990 workdir += fGridWorkingDir;
991 if (generate) {
992 TObjString *os;
993 if (!fInputFiles) {
994 Error("CreateJDL()", "Define some input files for your analysis.");
995 error = kTRUE;
996 }
997 // Compose list of input files
998 // Check if output files were defined
999 if (!fOutputFiles.Length()) {
1000 Error("CreateJDL", "You must define at least one output file");
1001 error = kTRUE;
1002 }
1003 // Check if an output directory was defined and valid
1004 if (!fGridOutputDir.Length()) {
1005 Error("CreateJDL", "You must define AliEn output directory");
1006 error = kTRUE;
1007 } else {
d3b18c4c 1008 if (!fProductionMode) {
1009 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1010 if (!DirectoryExists(fGridOutputDir)) {
1011 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1012 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1013 } else {
1014 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1015 // error = kTRUE;
1016 }
67a3b254 1017 } else {
1018 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1019 }
1020 gGrid->Cd(workdir);
1021 }
c57f56b7 1022 }
1023 // Exit if any error up to now
1024 if (error) return kFALSE;
1025 // Set JDL fields
0f389141 1026 if (!fUser.IsNull()) {
1027 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1028 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1029 }
1030 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1031 TString mergeExec = fExecutable;
1032 mergeExec.ReplaceAll(".sh", "_merge.sh");
1033 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1034 mergeExec.ReplaceAll(".sh", ".C");
1035 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1036 if (!fArguments.IsNull())
1037 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1038 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1039 else {
1040 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1041 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1042 }
b3e07543 1043
1f0d1ca2 1044 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1045 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1046 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048
0f389141 1049 if (fMaxInitFailed > 0) {
c57f56b7 1050 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1051 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1052 }
1053 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1054 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1055 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1056 }
1057 if (!IsOneStageMerging()) {
f47d5cb4 1058 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1059 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1060 }
1061 if (fSplitMode.Length()) {
c57f56b7 1062 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1063 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1064 }
1065 fMergingJDL->SetValue("Split", "\"se\"");
1066 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1067 if (!fAliROOTVersion.IsNull()) {
1068 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1069 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1070 }
1071 if (!fROOTVersion.IsNull()) {
c57f56b7 1072 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1073 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1074 }
1075 if (!fAPIVersion.IsNull()) {
c57f56b7 1076 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1077 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1078 }
648174cf 1079 if (!fExternalPackages.IsNull()) {
1080 arr = fExternalPackages.Tokenize(" ");
1081 TIter next(arr);
1082 while ((os=(TObjString*)next())) {
1083 TString pkgname = os->GetString();
1084 Int_t index = pkgname.Index("::");
1085 TString pkgversion = pkgname(index+2, pkgname.Length());
1086 pkgname.Remove(index);
1087 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1088 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1089 }
1090 delete arr;
1091 }
0f389141 1092 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1093 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1094 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1096 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1097 TString analysisFile = fExecutable;
1098 analysisFile.ReplaceAll(".sh", ".root");
1099 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1100 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1101 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1102 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1103 if (fAdditionalLibs.Length()) {
1104 arr = fAdditionalLibs.Tokenize(" ");
1105 TIter next(arr);
1106 while ((os=(TObjString*)next())) {
1107 if (os->GetString().Contains(".so")) continue;
1108 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1109 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1110 }
1111 delete arr;
1112 }
4e5c5506 1113 if (fPackages) {
1114 TIter next(fPackages);
1115 TObject *obj;
0f389141 1116 while ((obj=next())) {
4e5c5506 1117 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1118 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1119 }
4e5c5506 1120 }
c57f56b7 1121 if (fOutputArchive.Length()) {
1122 arr = fOutputArchive.Tokenize(" ");
1123 TIter next(arr);
0f389141 1124 Bool_t first = kTRUE;
1125 const char *comment = "Files to be archived";
1126 const char *comment1 = comment;
1127 while ((os=(TObjString*)next())) {
1128 if (!first) comment = NULL;
1129 if (!os->GetString().Contains("@") && fCloseSE.Length())
1130 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1131 else
1132 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1133 first = kFALSE;
1134 }
c57f56b7 1135 delete arr;
f790bc1b 1136 // Output archive for the merging jdl
1137 TString outputArchive;
1138 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1139 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1140 // Add normal output files, extra files + terminate files
1141 TString files = GetListOfFiles("outextter");
1142 // Do not register merge excludes
1143 if (!fMergeExcludes.IsNull()) {
1144 arr = fMergeExcludes.Tokenize(" ");
1145 TIter next1(arr);
1146 while ((os=(TObjString*)next1())) {
1147 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1148 files.ReplaceAll(os->GetString(),"");
1149 }
66d71516 1150 delete arr;
0f389141 1151 }
f790bc1b 1152 files.ReplaceAll(".root", "*.root");
b83f84f0 1153 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1154 } else {
95f84779 1155 TString files = fOutputArchive;
1156 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1157 outputArchive = files;
f790bc1b 1158 }
0f389141 1159 arr = outputArchive.Tokenize(" ");
705adb3e 1160 TIter next2(arr);
0f389141 1161 comment = comment1;
1162 first = kTRUE;
705adb3e 1163 while ((os=(TObjString*)next2())) {
0f389141 1164 if (!first) comment = NULL;
0cdf65a8 1165 TString currentfile = os->GetString();
0cdf65a8 1166 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1167 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1168 else
0cdf65a8 1169 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1170 first = kFALSE;
1171 }
1172 delete arr;
c57f56b7 1173 }
149d288c 1174 arr = fOutputFiles.Tokenize(",");
c57f56b7 1175 TIter next(arr);
0f389141 1176 Bool_t first = kTRUE;
f790bc1b 1177 const char *comment = "Files to be saved";
43da816a 1178 while ((os=(TObjString*)next())) {
e1eaf596 1179 // Ignore ouputs in jdl that are also in outputarchive
1180 TString sout = os->GetString();
f790bc1b 1181 sout.ReplaceAll("*", "");
1182 sout.ReplaceAll(".root", "");
e1eaf596 1183 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1184 if (fOutputArchive.Contains(sout)) continue;
0f389141 1185 if (!first) comment = NULL;
43da816a 1186 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1187 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1188 else
0f389141 1189 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1190 first = kFALSE;
f790bc1b 1191 if (fMergeExcludes.Contains(sout)) continue;
1192 if (!os->GetString().Contains("@") && fCloseSE.Length())
1193 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1194 else
1195 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1196 }
c57f56b7 1197 delete arr;
0f389141 1198 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1199 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1200 TString validationScript = fValidationScript;
0f389141 1201 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1202 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1203 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1204 if (fMasterResubmitThreshold) {
1205 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1206 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1207 }
d2a409b2 1208 // Write a jdl with 2 input parameters: collection name and output dir name.
1209 WriteJDL(copy);
c57f56b7 1210 }
1211 // Copy jdl to grid workspace
a8739e8a 1212 if (copy) {
b5e4aaa7 1213 // Check if an output directory was defined and valid
1214 if (!fGridOutputDir.Length()) {
1215 Error("CreateJDL", "You must define AliEn output directory");
1216 return kFALSE;
1217 } else {
1218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1219 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1220 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1221 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1222 } else {
1223 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 }
1227 gGrid->Cd(workdir);
1228 }
648174cf 1229 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1230 TString mergeJDLName = fExecutable;
1231 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1232 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1233 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1234 if (fProductionMode) {
648174cf 1235 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1236 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1237 }
648174cf 1238 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1239 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1240 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1241 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1242 if (fMergeViaJDL) {
1243 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1244 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1245 }
648174cf 1246 }
c57f56b7 1247 if (fAdditionalLibs.Length()) {
e7c71df0 1248 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
c57f56b7 1252 if (os->GetString().Contains(".so")) continue;
4e5c5506 1253 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1254 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1255 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1256 }
1257 delete arr;
1258 }
4e5c5506 1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 while ((obj=next())) {
fdbbc7be 1263 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1265 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1266 }
1267 }
c57f56b7 1268 }
1269 return kTRUE;
1270}
1271
a8739e8a 1272//______________________________________________________________________________
d2a409b2 1273Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1274{
1275// Writes one or more JDL's corresponding to findex. If findex is negative,
1276// all run numbers are considered in one go (jdl). For non-negative indices
1277// they correspond to the indices in the array fInputFiles.
1278 if (!fInputFiles) return kFALSE;
d3b18c4c 1279 TObject *os;
1280 TString workdir;
1281 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1282 workdir += fGridWorkingDir;
f47d5cb4 1283 TString stageName = "$2";
1284 if (fProductionMode) stageName = "$4";
1285 if (!fMergeDirName.IsNull()) {
1286 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1287 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1288 } else {
1289 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1290 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1291 }
d3b18c4c 1292 if (fProductionMode) {
a8739e8a 1293 TIter next(fInputFiles);
b3e07543 1294 while ((os=next())) {
d3b18c4c 1295 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1296 }
d3b18c4c 1297 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1298 } else {
1299 if (!fRunNumbers.Length() && !fRunRange[0]) {
1300 // One jdl with no parameters in case input data is specified by name.
1301 TIter next(fInputFiles);
1302 while ((os=next()))
1303 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1304 if (!fOutputSingle.IsNull())
1305 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1306 else {
1307 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1308 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1309 }
1310 } else {
1311 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1312 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1313 if (!fOutputSingle.IsNull()) {
1314 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1315 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1316 } else {
1317 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1318 }
1319 }
a8739e8a 1320 }
1321
a8739e8a 1322 // Generate the JDL as a string
1323 TString sjdl = fGridJDL->Generate();
0f389141 1324 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1325 // Final merge jdl
1326 if (!fMergeDirName.IsNull()) {
1327 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1328 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1329 } else {
1330 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1331 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1332 }
b3e07543 1333 TString sjdl2 = fMergingJDL->Generate();
1334 Int_t index, index1;
a8739e8a 1335 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1336 sjdl.ReplaceAll("(member", "\n (member");
1337 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1338 sjdl.ReplaceAll("{", "{\n ");
1339 sjdl.ReplaceAll("};", "\n};");
1340 sjdl.ReplaceAll("{\n \n", "{\n");
1341 sjdl.ReplaceAll("\n\n", "\n");
1342 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1343 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1344 sjdl1.ReplaceAll("(member", "\n (member");
1345 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1346 sjdl1.ReplaceAll("{", "{\n ");
1347 sjdl1.ReplaceAll("};", "\n};");
1348 sjdl1.ReplaceAll("{\n \n", "{\n");
1349 sjdl1.ReplaceAll("\n\n", "\n");
1350 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1351 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1352 sjdl2.ReplaceAll("(member", "\n (member");
1353 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1354 sjdl2.ReplaceAll("{", "{\n ");
1355 sjdl2.ReplaceAll("};", "\n};");
1356 sjdl2.ReplaceAll("{\n \n", "{\n");
1357 sjdl2.ReplaceAll("\n\n", "\n");
1358 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1359 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1360 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1361 index = sjdl.Index("JDLVariables");
1362 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1363 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1364 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1365 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1366 index = fJobTag.Index(":");
1367 if (index < 0) index = fJobTag.Length();
1368 TString jobTag = fJobTag;
f47d5cb4 1369 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1370 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1371 if (fProductionMode) {
1372 sjdl1.Prepend("# Generated merging jdl (production mode) \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = train number \
1375 \n# $3 = production (like LHC10b) \
1376 \n# $4 = merging stage \
1377 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1378 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1379 sjdl2.Prepend("# Generated merging jdl \
1380 \n# $1 = full alien path to output directory to be merged \
1381 \n# $2 = train number \
1382 \n# $3 = production (like LHC10b) \
1383 \n# $4 = merging stage \
1384 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1385 } else {
1386 sjdl1.Prepend("# Generated merging jdl \
1387 \n# $1 = full alien path to output directory to be merged \
1388 \n# $2 = merging stage \
03335324 1389 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1390 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1391 sjdl2.Prepend("# Generated merging jdl \
1392 \n# $1 = full alien path to output directory to be merged \
1393 \n# $2 = merging stage \
03335324 1394 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1395 }
0f389141 1396 index = sjdl1.Index("JDLVariables");
1397 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1398 index = sjdl2.Index("JDLVariables");
1399 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1400 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1401 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1402 index = sjdl2.Index("Split =");
1403 if (index>=0) {
1404 index1 = sjdl2.Index("\n", index);
1405 sjdl2.Remove(index, index1-index+1);
1406 }
1407 index = sjdl2.Index("SplitMaxInputFileNumber");
1408 if (index>=0) {
1409 index1 = sjdl2.Index("\n", index);
1410 sjdl2.Remove(index, index1-index+1);
1411 }
1412 index = sjdl2.Index("InputDataCollection");
1413 if (index>=0) {
1414 index1 = sjdl2.Index(";", index);
1415 sjdl2.Remove(index, index1-index+1);
1416 }
1417 index = sjdl2.Index("InputDataListFormat");
1418 if (index>=0) {
1419 index1 = sjdl2.Index("\n", index);
1420 sjdl2.Remove(index, index1-index+1);
1421 }
1422 index = sjdl2.Index("InputDataList");
1423 if (index>=0) {
1424 index1 = sjdl2.Index("\n", index);
1425 sjdl2.Remove(index, index1-index+1);
1426 }
f47d5cb4 1427 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1428 // Write jdl to file
a8739e8a 1429 ofstream out;
d2a409b2 1430 out.open(fJDLName.Data(), ios::out);
a8739e8a 1431 if (out.bad()) {
d3b18c4c 1432 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1433 return kFALSE;
1434 }
1435 out << sjdl << endl;
b3e07543 1436 out.close();
0f389141 1437 TString mergeJDLName = fExecutable;
1438 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1439 if (fMergeViaJDL) {
1440 ofstream out1;
1441 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1442 if (out1.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1444 return kFALSE;
1445 }
1446 out1 << sjdl1 << endl;
b3e07543 1447 out1.close();
1448 ofstream out2;
1449 TString finalJDL = mergeJDLName;
1450 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1451 out2.open(finalJDL.Data(), ios::out);
1452 if (out2.bad()) {
1453 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1454 return kFALSE;
1455 }
1456 out2 << sjdl2 << endl;
1457 out2.close();
0f389141 1458 }
a8739e8a 1459
1460 // Copy jdl to grid workspace
1461 if (!copy) {
d3b18c4c 1462 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1463 } else {
d2a409b2 1464 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1465 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1466 TString finalJDL = mergeJDLName;
1467 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1468 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1469 if (fProductionMode) {
b5fe9cba 1470 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1471 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1472 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1473 }
d2a409b2 1474 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1475 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1476 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1477 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1478 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1479 if (fMergeViaJDL) {
b3e07543 1480 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1481 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1482 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1483 }
a8739e8a 1484 }
1485 return kTRUE;
1486}
1487
c57f56b7 1488//______________________________________________________________________________
5513444a 1489Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1490{
1491// Returns true if file exists.
5513444a 1492 if (!gGrid) return kFALSE;
2ea55496 1493 TString slfn = lfn;
1494 slfn.ReplaceAll("alien://","");
1495 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1496 if (!res) return kFALSE;
1497 TMap *map = dynamic_cast<TMap*>(res->At(0));
1498 if (!map) {
1499 delete res;
1500 return kFALSE;
1501 }
1502 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1503 if (!objs || !objs->GetString().Length()) {
1504 delete res;
1505 return kFALSE;
1506 }
1507 delete res;
1508 return kTRUE;
1509}
1510
923e2ca5 1511//______________________________________________________________________________
1512Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1513{
1514// Returns true if directory exists. Can be also a path.
1515 if (!gGrid) return kFALSE;
1516 // Check if dirname is a path
1517 TString dirstripped = dirname;
1518 dirstripped = dirstripped.Strip();
1519 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1520 TString dir = gSystem->BaseName(dirstripped);
1521 dir += "/";
1522 TString path = gSystem->DirName(dirstripped);
1523 TGridResult *res = gGrid->Ls(path, "-F");
1524 if (!res) return kFALSE;
1525 TIter next(res);
1526 TMap *map;
1527 TObject *obj;
1528 while ((map=dynamic_cast<TMap*>(next()))) {
1529 obj = map->GetValue("name");
1530 if (!obj) break;
1531 if (dir == obj->GetName()) {
1532 delete res;
1533 return kTRUE;
1534 }
1535 }
1536 delete res;
1537 return kFALSE;
1538}
1539
c57f56b7 1540//______________________________________________________________________________
a2f5fc01 1541void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1542{
1543// Check input data type.
a2f5fc01 1544 isCollection = kFALSE;
1545 isXml = kFALSE;
1546 useTags = kFALSE;
c57f56b7 1547 if (!gGrid) {
1548 Error("CheckDataType", "No connection to grid");
1549 return;
1550 }
a2f5fc01 1551 isCollection = IsCollection(lfn);
c57f56b7 1552 TString msg = "\n##### file: ";
1553 msg += lfn;
a2f5fc01 1554 if (isCollection) {
c57f56b7 1555 msg += " type: raw_collection;";
1556 // special treatment for collections
a2f5fc01 1557 isXml = kFALSE;
c57f56b7 1558 // check for tag files in the collection
1559 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1560 if (!res) {
1561 msg += " using_tags: No (unknown)";
23329835 1562 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1563 return;
1564 }
1565 const char* typeStr = res->GetKey(0, "origLFN");
1566 if (!typeStr || !strlen(typeStr)) {
1567 msg += " using_tags: No (unknown)";
23329835 1568 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1569 return;
1570 }
1571 TString file = typeStr;
a2f5fc01 1572 useTags = file.Contains(".tag");
1573 if (useTags) msg += " using_tags: Yes";
c57f56b7 1574 else msg += " using_tags: No";
23329835 1575 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1576 return;
1577 }
1578 TString slfn(lfn);
1579 slfn.ToLower();
a2f5fc01 1580 isXml = slfn.Contains(".xml");
1581 if (isXml) {
c57f56b7 1582 // Open xml collection and check if there are tag files inside
1583 msg += " type: xml_collection;";
1584 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1585 if (!coll) {
1586 msg += " using_tags: No (unknown)";
23329835 1587 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1588 return;
1589 }
1590 TMap *map = coll->Next();
1591 if (!map) {
1592 msg += " using_tags: No (unknown)";
23329835 1593 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1594 return;
1595 }
1596 map = (TMap*)map->GetValue("");
1597 TString file;
1598 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1599 useTags = file.Contains(".tag");
c57f56b7 1600 delete coll;
a2f5fc01 1601 if (useTags) msg += " using_tags: Yes";
c57f56b7 1602 else msg += " using_tags: No";
23329835 1603 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1604 return;
1605 }
a2f5fc01 1606 useTags = slfn.Contains(".tag");
c57f56b7 1607 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1608 else msg += " type: unknown file;";
a2f5fc01 1609 if (useTags) msg += " using_tags: Yes";
c57f56b7 1610 else msg += " using_tags: No";
23329835 1611 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1612}
1613
4e5c5506 1614//______________________________________________________________________________
1615void AliAnalysisAlien::EnablePackage(const char *package)
1616{
1617// Enables a par file supposed to exist in the current directory.
1618 TString pkg(package);
1619 pkg.ReplaceAll(".par", "");
1620 pkg += ".par";
1621 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1622 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1623 return;
1624 }
1625 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1626 Info("EnablePackage", "AliEn plugin will use .par packages");
1627 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1628 if (!fPackages) {
1629 fPackages = new TObjArray();
1630 fPackages->SetOwner();
1631 }
1632 fPackages->Add(new TObjString(pkg));
1633}
1634
3bdcb562 1635//______________________________________________________________________________
1636TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1637{
1638// Make a tree from files having the location specified in fFileForTestMode.
1639// Inspired from JF's CreateESDChain.
1640 if (fFileForTestMode.IsNull()) {
1641 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1642 return NULL;
1643 }
1644 if (gSystem->AccessPathName(fFileForTestMode)) {
1645 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1646 return NULL;
1647 }
1648 // Open the file
1649 ifstream in;
1650 in.open(fFileForTestMode);
1651 Int_t count = 0;
1652 // Read the input list of files and add them to the chain
1653 TString line;
1654 TChain *chain = new TChain(treeName);
1655 while (in.good())
1656 {
1657 in >> line;
1658 if (line.IsNull()) continue;
1659 if (count++ == fNtestFiles) break;
1660 TString esdFile(line);
1661 TFile *file = TFile::Open(esdFile);
1662 if (file) {
1663 if (!file->IsZombie()) chain->Add(esdFile);
1664 file->Close();
1665 } else {
1666 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1667 }
1668 }
1669 in.close();
1670 if (!chain->GetListOfFiles()->GetEntries()) {
1671 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1672 delete chain;
1673 return NULL;
1674 }
1675// chain->ls();
1676 return chain;
1677}
1678
d2a409b2 1679//______________________________________________________________________________
1680const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1681{
1682// Get job status for all jobs with jobid>jobidstart.
1683 static char mstatus[20];
1684 mstatus[0] = '\0';
1685 nrunning = 0;
1686 nwaiting = 0;
1687 nerror = 0;
1688 ndone = 0;
1689 TGridJobStatusList *list = gGrid->Ps("");
1690 if (!list) return mstatus;
1691 Int_t nentries = list->GetSize();
1692 TGridJobStatus *status;
1693 Int_t pid;
1694 for (Int_t ijob=0; ijob<nentries; ijob++) {
1695 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1696 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1697 if (pid<jobidstart) continue;
1698 if (pid == lastid) {
8ddad121 1699 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1700 }
1701 switch (status->GetStatus()) {
1702 case TGridJobStatus::kWAITING:
1703 nwaiting++; break;
1704 case TGridJobStatus::kRUNNING:
1705 nrunning++; break;
1706 case TGridJobStatus::kABORTED:
1707 case TGridJobStatus::kFAIL:
1708 case TGridJobStatus::kUNKNOWN:
1709 nerror++; break;
1710 case TGridJobStatus::kDONE:
1711 ndone++;
1712 }
1713 }
1714 list->Delete();
1715 delete list;
1716 return mstatus;
1717}
1718
c57f56b7 1719//______________________________________________________________________________
1720Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1721{
1722// Returns true if file is a collection. Functionality duplicated from
1723// TAlien::Type() because we don't want to directly depend on TAlien.
1724 if (!gGrid) {
1725 Error("IsCollection", "No connection to grid");
1726 return kFALSE;
1727 }
1728 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1729 if (!res) return kFALSE;
1730 const char* typeStr = res->GetKey(0, "type");
1731 if (!typeStr || !strlen(typeStr)) return kFALSE;
1732 if (!strcmp(typeStr, "collection")) return kTRUE;
1733 delete res;
1734 return kFALSE;
1735}
1736
fe2d7fc2 1737//______________________________________________________________________________
1738Bool_t AliAnalysisAlien::IsSingleOutput() const
1739{
1740// Check if single-ouput option is on.
1741 return (!fOutputSingle.IsNull());
1742}
1743
16a4353c 1744//______________________________________________________________________________
1745void AliAnalysisAlien::Print(Option_t *) const
1746{
1747// Print current plugin settings.
84fcd93f 1748 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1750 if (mgr && mgr->IsProofMode()) {
1751 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1752 if (TestBit(AliAnalysisGrid::kTest))
1753 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1754 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1755 if (!fProofDataSet.IsNull())
1756 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1757 if (fProofReset==1)
1758 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1759 if (fProofReset>1)
1760 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (!fRootVersionForProof.IsNull())
1762 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1763 else
1764 printf("= ROOT version requested________________________ default\n");
1765 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1766 if (!fAliRootMode.IsNull())
1767 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1768 if (fNproofWorkers)
1769 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1770 if (fNproofWorkersPerSlave)
1771 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1772 if (TestSpecialBit(kClearPackages))
1773 printf("= ClearPackages requested...\n");
1774 if (fIncludePath.Data())
1775 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1776 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1777 if (fPackages && fPackages->GetEntries()) {
1778 TIter next(fPackages);
1779 TObject *obj;
1780 TString list;
1781 while ((obj=next())) list += obj->GetName();
1782 printf("= Par files to be used: ________________________ %s\n", list.Data());
1783 }
1784 if (TestSpecialBit(kProofConnectGrid))
1785 printf("= Requested PROOF connection to grid\n");
1786 return;
1787 }
e1c22e21 1788 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1789 if (fOverwriteMode) {
1790 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1791 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1792 }
348be253 1793 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1794 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1795 printf("= Production mode:______________________________ %d\n", fProductionMode);
1796 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1797 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1798 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1799 if (fUser.Length())
84fcd93f 1800 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1801 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1802 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1803 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1804 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1805 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1806 if (fRunNumbers.Length())
84fcd93f 1807 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1808 if (fRunRange[0])
74013535 1809 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1810 if (!fRunRange[0] && !fRunNumbers.Length()) {
1811 TIter next(fInputFiles);
1812 TObject *obj;
1813 TString list;
1814 while ((obj=next())) list += obj->GetName();
84fcd93f 1815 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1816 }
1817 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1818 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1819 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1820 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1821 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1822 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1823 printf("=====================================================================\n");
1824 printf("= Job price: ___________________________________ %d\n", fPrice);
1825 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1826 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1827 if (fMaxInitFailed>0)
84fcd93f 1828 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1829 if (fMasterResubmitThreshold>0)
84fcd93f 1830 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1831 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1832 if (fNrunsPerMaster>0)
84fcd93f 1833 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1834 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1835 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1836 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1837 if (fArguments.Length())
84fcd93f 1838 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1839 if (fExecutableArgs.Length())
1840 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1841 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1842 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1843 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1844 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1845 if (fDatasetName)
84fcd93f 1846 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1847 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1848 if (fIncludePath.Data())
84fcd93f 1849 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1850 if (fCloseSE.Length())
84fcd93f 1851 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1852 if (fFriendChainName.Length())
84fcd93f 1853 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1854 if (fPackages && fPackages->GetEntries()) {
16a4353c 1855 TIter next(fPackages);
1856 TObject *obj;
1857 TString list;
1858 while ((obj=next())) list += obj->GetName();
84fcd93f 1859 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1860 }
1861}
1862
c57f56b7 1863//______________________________________________________________________________
1864void AliAnalysisAlien::SetDefaults()
1865{
1866// Set default values for everything. What cannot be filled will be left empty.
1867 if (fGridJDL) delete fGridJDL;
1868 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1869 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1870 fPrice = 1;
1871 fTTL = 30000;
1872 fSplitMaxInputFileNumber = 100;
1873 fMaxInitFailed = 0;
1874 fMasterResubmitThreshold = 0;
bb885a9e 1875 fNtestFiles = 10;
149d288c 1876 fNreplicas = 2;
a8739e8a 1877 fRunRange[0] = 0;
1878 fRunRange[1] = 0;
8e3bd06a 1879 fRunPrefix = "%d";
319593fb 1880 fNrunsPerMaster = 1;
16a4353c 1881 fMaxMergeFiles = 100;
c57f56b7 1882 fRunNumbers = "";
1883 fExecutable = "analysis.sh";
0a1c1f7f 1884 fExecutableCommand = "root -b -q";
c57f56b7 1885 fArguments = "";
631c0b05 1886 fExecutableArgs = "";
c57f56b7 1887 fAnalysisMacro = "myAnalysis.C";
1888 fAnalysisSource = "";
1889 fAdditionalLibs = "";
1890 fSplitMode = "se";
1891 fAPIVersion = "";
1892 fROOTVersion = "";
1893 fAliROOTVersion = "";
1894 fUser = ""; // Your alien user name
1895 fGridWorkingDir = "";
1896 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1897 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1898 fFriendChainName = "";
c57f56b7 1899 fGridOutputDir = "output";
b83f84f0 1900 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1901 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1902 fInputFormat = "xml-single";
1903 fJDLName = "analysis.jdl";
c6cb3634 1904 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1905 fMergeExcludes = "";
0f389141 1906 fMergeViaJDL = 0;
348be253 1907 SetUseCopy(kTRUE);
1908 SetCheckCopy(kTRUE);
149d288c 1909 SetDefaultOutputs(kTRUE);
e1c22e21 1910 fOverwriteMode = 1;
c57f56b7 1911}
1912
0f389141 1913//______________________________________________________________________________
b3e07543 1914Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1915{
b3e07543 1916// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1917 // First check if the result is already in the output directory.
1918 if (FileExists(Form("%s/%s",aliendir,filename))) {
1919 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1920 return kFALSE;
1921 }
b3e07543 1922 // Now check the last stage done.
1923 Int_t stage = 0;
1924 while (1) {
1925 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1926 stage++;
1927 }
1928 // Next stage of merging
1929 stage++;
1930 TString pattern = "*root_archive.zip";
1931 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1932 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1933 if (res) delete res;
1934 // Write standard output to file
1935 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1936 // Count the number of files inside
1937 ifstream ifile;
1938 ifile.open(Form("Stage_%d.xml",stage));
1939 if (!ifile.good()) {
1940 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1941 return kFALSE;
b3e07543 1942 }
1943 TString line;
1944 Int_t nfiles = 0;
1945 while (!ifile.eof()) {
1946 ifile >> line;
1947 if (line.Contains("/event")) nfiles++;
7c2cd90a 1948 }
b3e07543 1949 ifile.close();
1950 if (!nfiles) {
1951 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1952 return kFALSE;
1953 } else {
1954 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1955 }
b3e07543 1956 // Copy the file in the output directory
1957 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1958 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1959 // Check if this is the last stage to be done.
1960 Bool_t laststage = (nfiles<nperchunk);
1961 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1962 if (laststage) {
1963 printf("### Submiting final merging stage %d\n", stage);
1964 TString finalJDL = jdl;
1965 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1966 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1967 Int_t jobId = SubmitSingleJob(query);
1968 if (!jobId) return kFALSE;
1969 } else {
1970 printf("### Submiting merging stage %d\n", stage);
03335324 1971 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1972 Int_t jobId = SubmitSingleJob(query);
1973 if (!jobId) return kFALSE;
7c2cd90a 1974 }
b3e07543 1975 return kTRUE;
1976}
7c2cd90a 1977
1978//______________________________________________________________________________
1979Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1980{
1981// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1982 if (!gGrid) return 0;
1983 printf("=> %s ------> ",query);
1984 TGridResult *res = gGrid->Command(query);
1985 if (!res) return 0;
1986 TString jobId = res->GetKey(0,"jobId");
1987 delete res;
1988 if (jobId.IsNull()) {
1989 printf("submission failed. Reason:\n");
1990 gGrid->Stdout();
1991 gGrid->Stderr();
1992 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1993 return 0;
1994 }
1995 printf(" Job id: %s\n", jobId.Data());
1996 return atoi(jobId);
1997}
1998
1999//______________________________________________________________________________
b3e07543 2000Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2001{
b3e07543 2002// Merge given output files from basedir. Basedir can be an alien output directory
2003// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2004// files in a group (ignored for xml input). Merging can be done in stages:
2005// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2006// stage=1 : works with an xml of all root_archive.zip in the output directory
2007// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2008 TString outputFile = output;
0f389141 2009 TString command;
a2f5fc01 2010 TString outputChunk;
2011 TString previousChunk = "";
b3e07543 2012 TObjArray *listoffiles = new TObjArray();
2013// listoffiles->SetOwner();
a2f5fc01 2014 Int_t countChunk = 0;
2015 Int_t countZero = nmaxmerge;
0f389141 2016 Bool_t merged = kTRUE;
a2f5fc01 2017 Int_t index = outputFile.Index("@");
2018 if (index > 0) outputFile.Remove(index);
7c2cd90a 2019 TString inputFile = outputFile;
b3e07543 2020 TString sbasedir = basedir;
2021 if (sbasedir.Contains(".xml")) {
2022 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2023 nmaxmerge = 9999999;
2024 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2025 if (!coll) {
2026 ::Error("MergeOutput", "Input XML collection empty.");
2027 return kFALSE;
2028 }
2029 // Iterate grid collection
2030 while (coll->Next()) {
2031 TString fname = gSystem->DirName(coll->GetTURL());
2032 fname += "/";
2033 fname += inputFile;
2034 listoffiles->Add(new TNamed(fname.Data(),""));
2035 }
2036 } else {
2037 command = Form("find %s/ *%s", basedir, inputFile.Data());
2038 printf("command: %s\n", command.Data());
2039 TGridResult *res = gGrid->Command(command);
2040 if (!res) {
2041 ::Error("MergeOutput","No result for the find command\n");
2042 delete listoffiles;
2043 return kFALSE;
2044 }
2045 TIter nextmap(res);
2046 TMap *map = 0;
2047 while ((map=(TMap*)nextmap())) {
2048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2049 if (!objs || !objs->GetString().Length()) {
2050 // Nothing found - skip this output
2051 delete res;
2052 delete listoffiles;
2053 return kFALSE;
2054 }
2055 listoffiles->Add(new TNamed(objs->GetName(),""));
2056 }
2057 delete res;
2058 }
2059 if (!listoffiles->GetEntries()) {
7c2cd90a 2060 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2061 delete listoffiles;
0f389141 2062 return kFALSE;
2063 }
2064
2065 TFileMerger *fm = 0;
b3e07543 2066 TIter next0(listoffiles);
2067 TObjArray *listoffilestmp = new TObjArray();
2068 listoffilestmp->SetOwner();
2069 TObject *nextfile;
2070 TString snextfile;
2071 // Keep only the files at upper level
2072 Int_t countChar = 0;
2073 while ((nextfile=next0())) {
2074 snextfile = nextfile->GetName();
2075 Int_t crtCount = snextfile.CountChar('/');
2076 if (nextfile == listoffiles->First()) countChar = crtCount;
2077 if (crtCount < countChar) countChar = crtCount;
2078 }
2079 next0.Reset();
2080 while ((nextfile=next0())) {
2081 snextfile = nextfile->GetName();
2082 Int_t crtCount = snextfile.CountChar('/');
2083 if (crtCount > countChar) {
2084 delete nextfile;
2085 continue;
2086 }
2087 listoffilestmp->Add(nextfile);
2088 }
2089 delete listoffiles;
2090 listoffiles = listoffilestmp; // Now contains 'good' files
2091 listoffiles->Print();
2092 TIter next(listoffiles);
7c2cd90a 2093 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2094 outputChunk = outputFile;
2095 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2096 // Check for existent temporary merge files
e1c22e21 2097 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2098 // Preserve old merging functionality for stage 0.
2099 if (stage==0) {
2100 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2101 while (1) {
2102 // Skip as many input files as in a chunk
2ea55496 2103 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2104 nextfile = next();
2105 if (!nextfile) {
2ea55496 2106 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2107 delete listoffiles;
2ea55496 2108 return kFALSE;
2109 }
b3e07543 2110 snextfile = nextfile->GetName();
7c2cd90a 2111 }
2112 outputChunk = outputFile;
2113 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2114 countChunk++;
2115 if (gSystem->AccessPathName(outputChunk)) continue;
2116 // Merged file with chunks up to <countChunk> found
2117 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2118 previousChunk = outputChunk;
2119 break;
2120 }
2121 }
2122 countZero = nmaxmerge;
2123
b3e07543 2124 while ((nextfile=next())) {
2125 snextfile = nextfile->GetName();
2ea55496 2126 // Loop 'find' results and get next LFN
7c2cd90a 2127 if (countZero == nmaxmerge) {
2128 // First file in chunk - create file merger and add previous chunk if any.
2129 fm = new TFileMerger(kFALSE);
2130 fm->SetFastMethod(kTRUE);
2131 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2132 outputChunk = outputFile;
2133 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2134 }
2135 // If last file found, put merged results in the output file
b3e07543 2136 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2137 // Add file to be merged and decrement chunk counter.
b3e07543 2138 fm->AddFile(snextfile);
7c2cd90a 2139 countZero--;
b3e07543 2140 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2141 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2142 // Nothing found - skip this output
2143 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2144 merged = kFALSE;
2145 break;
7c2cd90a 2146 }
2147 fm->OutputFile(outputChunk);
2148 // Merge the outputs, then go to next chunk
2149 if (!fm->Merge()) {
2150 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2151 merged = kFALSE;
2152 break;
7c2cd90a 2153 } else {
2154 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2155 gSystem->Unlink(previousChunk);
2156 }
b3e07543 2157 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2158 countChunk++;
2159 countZero = nmaxmerge;
2160 previousChunk = outputChunk;
0f389141 2161 }
0f389141 2162 }
b3e07543 2163 delete listoffiles;
2164 delete fm;
7c2cd90a 2165 return merged;
2166 }
2167 // Merging stage different than 0.
2168 // Move to the begining of the requested chunk.
7c2cd90a 2169 fm = new TFileMerger(kFALSE);
2170 fm->SetFastMethod(kTRUE);
b3e07543 2171 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2172 delete listoffiles;
7c2cd90a 2173 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2174 // Nothing found - skip this output
2175 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2176 delete fm;
2177 return kFALSE;
2178 }
b3e07543 2179 fm->OutputFile(outputFile);
7c2cd90a 2180 // Merge the outputs
2181 if (!fm->Merge()) {
2182 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2183 delete fm;
2184 return kFALSE;
2185 } else {
b3e07543 2186 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2187 }
7c2cd90a 2188 delete fm;
2189 return kTRUE;
0f389141 2190}
2191
c57f56b7 2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::MergeOutputs()
2194{
2195// Merge analysis outputs existing in the AliEn space.
2196 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2197 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2198 if (!Connect()) {
2199 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2200 return kFALSE;
0f389141 2201 }
c9e8f7fd 2202 if (fMergeViaJDL) {
2203 if (!TestBit(AliAnalysisGrid::kMerge)) {
2204 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2205 return kFALSE;
2206 }
2207 if (fProductionMode) {
2208 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2209 return kFALSE;
2210 }
0f389141 2211 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2212 if (!SubmitMerging()) return kFALSE;
0f389141 2213 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2214 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2215 return kFALSE;
c57f56b7 2216 }
2217 // Get the output path
d2a409b2 2218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2219 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2220 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2221 return kFALSE;
2222 }
2223 if (!fOutputFiles.Length()) {
2224 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2225 return kFALSE;
0f389141 2226 }
2227 // Check if fast read option was requested
e1c22e21 2228 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2229 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2230 if (fFastReadOption) {
d3339be3 2231 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2232 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2233 gEnv->SetValue("XNet.ConnectTimeout",50);
2234 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2235 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2236 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2237 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2238 }
e8b839ab 2239 // Make sure we change the temporary directory
2240 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2241 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2242 TIter next(list);
2243 TObjString *str;
a2f5fc01 2244 TString outputFile;
c57f56b7 2245 Bool_t merged = kTRUE;
2246 while((str=(TObjString*)next())) {
a2f5fc01 2247 outputFile = str->GetString();
2248 Int_t index = outputFile.Index("@");
2249 if (index > 0) outputFile.Remove(index);
2250 TString outputChunk = outputFile;
2251 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2252 // Skip already merged outputs
a2f5fc01 2253 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2254 if (fOverwriteMode) {
a2f5fc01 2255 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2256 gSystem->Unlink(outputFile);
2257 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2258 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2259 outputChunk.Data());
2260 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2261 }
2262 } else {
a2f5fc01 2263 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2264 continue;
2265 }
2266 } else {
a2f5fc01 2267 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2268 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2269 outputChunk.Data());
2270 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2271 }
2272 }
bb885a9e 2273 if (fMergeExcludes.Length() &&
a2f5fc01 2274 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2275 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2276 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2277 if (!merged) {
2278 Error("MergeOutputs", "Terminate() will NOT be executed");
2279 return kFALSE;
ff07ec61 2280 }
2281 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2282 if (fileOpened) fileOpened->Close();
c57f56b7 2283 }
0f389141 2284 return kTRUE;
c57f56b7 2285}
2286
bb885a9e 2287//______________________________________________________________________________
2288void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2289{
2290// Use the output files connected to output containers from the analysis manager
2291// rather than the files defined by SetOutputFiles
2292 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2293 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2294 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2295}
2296
149d288c 2297//______________________________________________________________________________
2298void AliAnalysisAlien::SetOutputFiles(const char *list)
2299{
2300// Manually set the output files list.
2301// Removes duplicates. Not allowed if default outputs are not disabled.
2302 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2303 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2304 return;
2305 }
2306 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2307 fOutputFiles = "";
2308 TString slist = list;
2309 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2310 TObjArray *arr = slist.Tokenize(" ");
2311 TObjString *os;
2312 TIter next(arr);
2313 TString sout;
2314 while ((os=(TObjString*)next())) {
2315 sout = os->GetString();
2316 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2317 if (fOutputFiles.Contains(sout)) continue;
2318 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2319 fOutputFiles += sout;
2320 }
2321 delete arr;
f790bc1b 2322}
149d288c 2323
2324//______________________________________________________________________________
2325void AliAnalysisAlien::SetOutputArchive(const char *list)
2326{
2327// Manually set the output archive list. Free text - you are on your own...
2328// Not allowed if default outputs are not disabled.
2329 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2330 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2331 return;
2332 }
2333 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2334 fOutputArchive = list;
2335}
2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2339{
2340// Setting a prefered output SE is not allowed anymore.
2341 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2342}
2343
5513444a 2344//______________________________________________________________________________
2345Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2346{
2347// Start remote grid analysis.
3bdcb562 2348 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2349 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2350 if (!mgr || !mgr->IsInitialized()) {
2351 Error("StartAnalysis", "You need an initialized analysis manager for this");
2352 return kFALSE;
2353 }
2354 // Are we in PROOF mode ?
2355 if (mgr->IsProofMode()) {
90d50a8c 2356 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2357 if (fProofCluster.IsNull()) {
2358 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2359 return kFALSE;
2360 }
2361 if (fProofDataSet.IsNull() && !testMode) {
2362 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2363 return kFALSE;
2364 }
2365 // Set the needed environment
2366 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2367 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2368 if (fProofReset && !testMode) {
2369 if (fProofReset==1) {
2370 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2371 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2372 } else {
2373 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2374 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2375 }
2376 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2377 return kFALSE;
2378 }
2379 // Do we need to change the ROOT version ? The success of this cannot be checked.
2380 if (!fRootVersionForProof.IsNull() && !testMode) {
2381 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2382 fProofCluster.Data(), fRootVersionForProof.Data()));
2383 }
2384 // Connect to PROOF and check the status
2385 Long_t proof = 0;
27734f0e 2386 TString sworkers;
2387 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2388 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2389 if (!testMode) {
27734f0e 2390 if (!sworkers.IsNull())
2391 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2392 else
2393 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2394 } else {
2395 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2396 if (!proof) {
2397 Error("StartAnalysis", "Could not start PROOF in test mode");
2398 return kFALSE;
2399 }
2400 }
2401 if (!proof) {
2402 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2403 return kFALSE;
2404 }
27734f0e 2405 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2406 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2407 // Is dataset existing ?
2408 if (!testMode) {
2409 TString dataset = fProofDataSet;
2410 Int_t index = dataset.Index("#");
2411 if (index>=0) dataset.Remove(index);
830acc4c 2412// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2413// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2414// return kFALSE;
2415// }
2416// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2417 }
2418 // Is ClearPackages() needed ?
2419 if (TestSpecialBit(kClearPackages)) {
2420 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2421 gROOT->ProcessLine("gProof->ClearPackages();");
2422 }
2423 // Is a given aliroot mode requested ?
2424 TList optionsList;
1ac846e9 2425 TString parLibs;
830acc4c 2426 if (!fAliRootMode.IsNull()) {
3bdcb562 2427 TString alirootMode = fAliRootMode;
2428 if (alirootMode == "default") alirootMode = "";
2429 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2430 optionsList.SetOwner();
2431 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2432 // Check the additional libs to be loaded
2433 TString extraLibs;
1ac846e9 2434 Bool_t parMode = kFALSE;
3bdcb562 2435 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2436 // Parse the extra libs for .so
2437 if (fAdditionalLibs.Length()) {
2438 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2439 TIter next(list);
2440 TObjString *str;
1ac846e9 2441 while((str=(TObjString*)next())) {
2442 if (str->GetString().Contains(".so")) {
2443 if (parMode) {
2444 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2445 break;
2446 }
2447 TString stmp = str->GetName();
2448 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2449 stmp.ReplaceAll(".so","");
2450 if (!extraLibs.IsNull()) extraLibs += ":";
2451 extraLibs += stmp;
2452 continue;
2453 }
2454 if (str->GetString().Contains(".par")) {
2455 // The first par file found in the list will not allow any further .so
2456 parMode = kTRUE;
2457 if (!parLibs.IsNull()) parLibs += ":";
2458 parLibs += str->GetName();
2459 continue;
2460 }
3bdcb562 2461 }
2462 if (list) delete list;
2463 }
2464 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2465 // Check extra includes
2466 if (!fIncludePath.IsNull()) {
2467 TString includePath = fIncludePath;
2468 includePath.ReplaceAll(" ",":");
8ddad121 2469 includePath.ReplaceAll("$ALICE_ROOT","");
2470 includePath.ReplaceAll("${ALICE_ROOT}","");
2471 includePath.ReplaceAll("-I","");
830acc4c 2472 includePath.Strip(TString::kTrailing, ':');
2473 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2474 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2475 }
27734f0e 2476 // Check if connection to grid is requested
2477 if (TestSpecialBit(kProofConnectGrid))
2478 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2479 // Enable AliRoot par
2480 if (testMode) {
2481 // Enable proof lite package
2482 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2483 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2484 TNamed *obj = (TNamed*)optionsList.At(i);
2485 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2486 }
2487 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2488 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2489 Info("StartAnalysis", "AliRootProofLite enabled");
2490 } else {
2491 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2492 return kFALSE;
2493 }
2494 } else {
8ddad121 2495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2496 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2497 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2498 return kFALSE;
2499 }
27734f0e 2500 }
1ac846e9 2501 // Enable first par files from fAdditionalLibs
2502 if (!parLibs.IsNull()) {
2503 TObjArray *list = parLibs.Tokenize(":");
2504 TIter next(list);
2505 TObjString *package;
2506 while((package=(TObjString*)next())) {
40501bb7 2507 TString spkg = package->GetName();
2508 spkg.ReplaceAll(".par", "");
2509 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2510 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2511 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2512 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2513 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2514 return kFALSE;
2515 }
2516 } else {
2517 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2518 return kFALSE;
2519 }
2520 }
2521 if (list) delete list;
2522 }
3bdcb562 2523 } else {
2524 if (fAdditionalLibs.Contains(".so") && !testMode) {
2525 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2526 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2527 return kFALSE;
2528 }
2529 }
2530 // Enable par files if requested
2531 if (fPackages && fPackages->GetEntries()) {
2532 TIter next(fPackages);
2533 TObject *package;
2534 while ((package=next())) {
1ac846e9 2535 // Skip packages already enabled
2536 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2537 TString spkg = package->GetName();
2538 spkg.ReplaceAll(".par", "");
2539 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2540 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2541 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2542 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2543 return kFALSE;
2544 }
2545 } else {
2546 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2547 return kFALSE;
2548 }
2549 }
2550 }
2551 // Do we need to load analysis source files ?
2552 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2553 if (fAnalysisSource.Length()) {
2554 TObjArray *list = fAnalysisSource.Tokenize(" ");
2555 TIter next(list);
2556 TObjString *str;
2557 while((str=(TObjString*)next())) {
2558 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2559 }
2560 if (list) delete list;
2561 }
830acc4c 2562 if (testMode) {
2563 // Register dataset to proof lite.
2564 if (fFileForTestMode.IsNull()) {
2565 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2566 return kFALSE;
2567 }
2568 if (gSystem->AccessPathName(fFileForTestMode)) {
2569 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2570 return kFALSE;
2571 }
2572 TFileCollection *coll = new TFileCollection();
2573 coll->AddFromFile(fFileForTestMode);
8ddad121 2574 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2575 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2576 }
3bdcb562 2577 return kTRUE;
2578 }
c57f56b7 2579
43da816a 2580 // Check if output files have to be taken from the analysis manager
2581 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2582 // Add output files and AOD files
2583 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2584 // Add extra files registered to the analysis manager
f790bc1b 2585 TString extra = GetListOfFiles("ext");
2586 if (!extra.IsNull()) {
149d288c 2587 extra.ReplaceAll(".root", "*.root");
f790bc1b 2588 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2589 fOutputFiles += extra;
84fcd93f 2590 }
149d288c 2591 // Compose the output archive.
b83f84f0 2592 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2593 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2594 }
f7b1cbc2 2595// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2596 if (TestBit(AliAnalysisGrid::kOffline)) {
2597 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2598 \n there nor any job run. You can revise the JDL and analysis \
2599 \n macro then run the same in \"submit\" mode.");
2600 } else if (TestBit(AliAnalysisGrid::kTest)) {
2601 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2602 \n dataset.");
2603 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2604 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2605 \n space and job submitted.");
2606 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2607 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2608 if (fMergeViaJDL) CheckInputData();
5513444a 2609 return kTRUE;
c57f56b7 2610 } else {
2611 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2612 }
2613
348be253 2614 Print();
c57f56b7 2615 if (!Connect()) {
2616 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2617 return kFALSE;
16a4353c 2618 }
d3b18c4c 2619 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2620 if (!CheckInputData()) {
2621 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2622 return kFALSE;
c57f56b7 2623 }
d3339be3 2624 if (!CreateDataset(fDataPattern)) {
2625 TString serror;
2626 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2627 if (fRunNumbers.Length()) serror = "run numbers";
2628 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2629 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2630 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2631 return kFALSE;
2632 }
c57f56b7 2633 WriteAnalysisFile();
2634 WriteAnalysisMacro();
2635 WriteExecutable();
2636 WriteValidationScript();
0f389141 2637 if (fMergeViaJDL) {
2638 WriteMergingMacro();
2639 WriteMergeExecutable();
2640 WriteValidationScript(kTRUE);
2641 }
5513444a 2642 if (!CreateJDL()) return kFALSE;
2643 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2644 if (testMode) {
c57f56b7 2645 // Locally testing the analysis
2646 Info("StartAnalysis", "\n_______________________________________________________________________ \
2647 \n Running analysis script in a daughter shell as on a worker node \
2648 \n_______________________________________________________________________");
149d288c 2649 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2650 TIter next(list);
2651 TObjString *str;
a2f5fc01 2652 TString outputFile;
c57f56b7 2653 while((str=(TObjString*)next())) {
a2f5fc01 2654 outputFile = str->GetString();
2655 Int_t index = outputFile.Index("@");
2656 if (index > 0) outputFile.Remove(index);
2657 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2658 }
2659 delete list;
2660 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2661 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2662// gSystem->Exec("cat stdout");
5513444a 2663 return kFALSE;
c57f56b7 2664 }
5513444a 2665 // Check if submitting is managed by LPM manager
a3e84053 2666 if (fProductionMode) {
5513444a 2667 TString prodfile = fJDLName;
2668 prodfile.ReplaceAll(".jdl", ".prod");
2669 WriteProductionFile(prodfile);
2670 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2671 return kFALSE;
2672 }
a8739e8a 2673 // Submit AliEn job(s)
d2a409b2 2674 gGrid->Cd(fGridOutputDir);
a8739e8a 2675 TGridResult *res;
c57f56b7 2676 TString jobID = "";
d2a409b2 2677 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2678 // Submit a given xml or a set of runs
a8739e8a 2679 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2680 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2681 if (res) {
2682 const char *cjobId = res->GetKey(0,"jobId");
2683 if (!cjobId) {
a03be957 2684 gGrid->Stdout();
2685 gGrid->Stderr();
a8739e8a 2686 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2687 return kFALSE;
a8739e8a 2688 } else {
2689 Info("StartAnalysis", "\n_______________________________________________________________________ \
2690 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2691 \n_______________________________________________________________________",
2692 fJDLName.Data(), cjobId);
2693 jobID = cjobId;
2694 }
2695 delete res;
a03be957 2696 } else {
2697 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2698 return kFALSE;
a8739e8a 2699 }
2700 } else {
d2a409b2 2701 // Submit for a range of enumeration of runs.
a03be957 2702 if (!Submit()) return kFALSE;
c57f56b7 2703 }
a8739e8a 2704
c57f56b7 2705 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2706 \n You may exit at any time and terminate the job later using the option <terminate> \
2707 \n ##################################################################################", jobID.Data());
bb885a9e 2708 gSystem->Exec("aliensh");
5513444a 2709 return kTRUE;
c57f56b7 2710}
2711
f790bc1b 2712//______________________________________________________________________________
2713const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2714{
2715// Get a comma-separated list of output files of the requested type.
2716// Type can be (case unsensitive):
2717// aod - list of aod files (std, extensions and filters)
2718// out - list of output files connected to containers (but not aod's or extras)
2719// ext - list of extra files registered to the manager
2720// ter - list of files produced in terminate
2721 static TString files;
2722 files = "";
2723 TString stype = type;
2724 stype.ToLower();
2725 TString aodfiles, extra;
2726 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2727 if (!mgr) {
2728 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2729 return files.Data();
2730 }
2731 if (mgr->GetOutputEventHandler()) {
2732 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2733 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2734 if (!extraaod.IsNull()) {
2735 aodfiles += ",";
2736 aodfiles += extraaod;
2737 }
2738 }
2739 if (stype.Contains("aod")) {
2740 files = aodfiles;
2741 if (stype == "aod") return files.Data();
2742 }
2743 // Add output files that are not in the list of AOD files
2744 TString outputfiles = "";
2745 TIter next(mgr->GetOutputs());
2746 AliAnalysisDataContainer *output;
2747 const char *filename = 0;
2748 while ((output=(AliAnalysisDataContainer*)next())) {
2749 filename = output->GetFileName();
2750 if (!(strcmp(filename, "default"))) continue;
2751 if (outputfiles.Contains(filename)) continue;
2752 if (aodfiles.Contains(filename)) continue;
2753 if (!outputfiles.IsNull()) outputfiles += ",";
2754 outputfiles += filename;
2755 }
2756 if (stype.Contains("out")) {
2757 if (!files.IsNull()) files += ",";
2758 files += outputfiles;
2759 if (stype == "out") return files.Data();
2760 }
2761 // Add extra files registered to the analysis manager
2762 TString sextra;
2763 extra = mgr->GetExtraFiles();
2764 if (!extra.IsNull()) {
2765 extra.Strip();
2766 extra.ReplaceAll(" ", ",");
2767 TObjArray *fextra = extra.Tokenize(",");
2768 TIter nextx(fextra);
2769 TObject *obj;
2770 while ((obj=nextx())) {
2771 if (aodfiles.Contains(obj->GetName())) continue;
2772 if (outputfiles.Contains(obj->GetName())) continue;
2773 if (sextra.Contains(obj->GetName())) continue;
2774 if (!sextra.IsNull()) sextra += ",";
2775 sextra += obj->GetName();
2776 }
2777 delete fextra;
2778 if (stype.Contains("ext")) {
2779 if (!files.IsNull()) files += ",";
2780 files += sextra;
2781 }
2782 }
2783 if (stype == "ext") return files.Data();
2784 TString termfiles;
2785 if (!fTerminateFiles.IsNull()) {
2786 fTerminateFiles.Strip();
2787 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2788 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2789 TIter nextx(fextra);
2790 TObject *obj;
2791 while ((obj=nextx())) {
2792 if (aodfiles.Contains(obj->GetName())) continue;
2793 if (outputfiles.Contains(obj->GetName())) continue;
2794 if (termfiles.Contains(obj->GetName())) continue;
2795 if (sextra.Contains(obj->GetName())) continue;
2796 if (!termfiles.IsNull()) termfiles += ",";
2797 termfiles += obj->GetName();
2798 }
2799 delete fextra;
2800 }
2801 if (stype.Contains("ter")) {
2802 if (!files.IsNull() && !termfiles.IsNull()) {
2803 files += ",";
2804 files += termfiles;
2805 }
2806 }
2807 return files.Data();
2808}
2809
d2a409b2 2810//______________________________________________________________________________
a03be957 2811Bool_t AliAnalysisAlien::Submit()
d2a409b2 2812{
2813// Submit all master jobs.
2814 Int_t nmasterjobs = fInputFiles->GetEntries();
2815 Long_t tshoot = gSystem->Now();
a03be957 2816 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2817 while (fNsubmitted < nmasterjobs) {
2818 Long_t now = gSystem->Now();
2819 if ((now-tshoot)>30000) {
2820 tshoot = now;
a03be957 2821 if (!SubmitNext()) return kFALSE;
d2a409b2 2822 }
2823 }
a03be957 2824 return kTRUE;
d2a409b2 2825}
2826
0f389141 2827//______________________________________________________________________________
a03be957 2828Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2829{
2830// Submit all merging jobs.
2831 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2832 gGrid->Cd(fGridOutputDir);
2833 TString mergeJDLName = fExecutable;
2834 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2835 if (!fInputFiles) {
2836 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2837 return kFALSE;
2838 }
0f389141 2839 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2840 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2841 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2842 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2843 if (fOutputToRunNo) {
2844 // The output directory is the run number
2845 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2846 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2847 } else {
2848 // The output directory is the master number in 3 digits format
2849 printf("### Submitting merging job for master <%03d>\n", i);
2850 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2851 }
2852 // Check now the number of merging stages.
2b222f7c 2853 TObjArray *list = fOutputFiles.Tokenize(",");
2854 TIter next(list);
2855 TObjString *str;
2856 TString outputFile;
2857 while((str=(TObjString*)next())) {
2858 outputFile = str->GetString();
2859 Int_t index = outputFile.Index("@");
2860 if (index > 0) outputFile.Remove(index);
2861 if (!fMergeExcludes.Contains(outputFile)) break;
2862 }
2863 delete list;
b3e07543 2864 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2865 if (!done && (i==ntosubmit-1)) return kFALSE;
0f389141 2866 }
a03be957 2867 if (!ntosubmit) return kTRUE;
9f020952 2868 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2869 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2870 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2871 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2872 \n ################################################################################################################");
0f389141 2873 gSystem->Exec("aliensh");
a03be957 2874 return kTRUE;
0f389141 2875}
2876
d2a409b2 2877//______________________________________________________________________________
a03be957 2878Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2879{
08d5b699 2880// Submit next bunch of master jobs if the queue is free. The first master job is
2881// submitted right away, while the next will not be unless the previous was split.
2882// The plugin will not submit new master jobs if there are more that 500 jobs in
2883// waiting phase.
d2a409b2 2884 static Bool_t iscalled = kFALSE;
2885 static Int_t firstmaster = 0;
2886 static Int_t lastmaster = 0;
2887 static Int_t npermaster = 0;
a03be957 2888 if (iscalled) return kTRUE;
d2a409b2 2889 iscalled = kTRUE;
2890 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2891 Int_t ntosubmit = 0;
2892 TGridResult *res;
2893 TString jobID = "";
3bdcb562 2894 Int_t nmasterjobs = fInputFiles->GetEntries();
2895 if (!fNsubmitted) {
2896 ntosubmit = 1;
2897 if (!IsUseSubmitPolicy()) {
90d50a8c 2898 if (nmasterjobs>5)
3bdcb562 2899 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2900 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2901 ntosubmit = nmasterjobs;
2902 }
2903 } else {
d2a409b2 2904 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2905 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2906 // If last master not split, just return
a03be957 2907 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2908 // No more than 100 waiting jobs
08d5b699 2909 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2910 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2911 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2912 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2913 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2914 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2915 }
d2a409b2 2916 for (Int_t i=0; i<ntosubmit; i++) {
2917 // Submit for a range of enumeration of runs.
a03be957 2918 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2919 TString query;
cd11251e 2920 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2921 runOutDir.ReplaceAll(".xml", "");
2922 if (fOutputToRunNo)
2923 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2924 else
2925 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2926 printf("********* %s\n",query.Data());
d2a409b2 2927 res = gGrid->Command(query);
2928 if (res) {
98ca124f 2929 TString cjobId1 = res->GetKey(0,"jobId");
2930 if (!cjobId1.Length()) {
d2a409b2 2931 iscalled = kFALSE;
a03be957 2932 gGrid->Stdout();
2933 gGrid->Stderr();
2934 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2935 return kFALSE;
d2a409b2 2936 } else {
2937 Info("StartAnalysis", "\n_______________________________________________________________________ \
2938 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2939 \n_______________________________________________________________________",
98ca124f 2940 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2941 jobID += cjobId1;
2942 jobID += " ";
98ca124f 2943 lastmaster = cjobId1.Atoi();
d2a409b2 2944 if (!firstmaster) firstmaster = lastmaster;
2945 fNsubmitted++;
2946 }
2947 delete res;
a03be957 2948 } else {
2949 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2950 return kFALSE;
d2a409b2 2951 }
2952 }
2953 iscalled = kFALSE;
a03be957 2954 return kTRUE;
d2a409b2 2955}
2956
c57f56b7 2957//______________________________________________________________________________
2958void AliAnalysisAlien::WriteAnalysisFile()
2959{
f10e8481 2960// Write current analysis manager into the file <analysisFile>
2961 TString analysisFile = fExecutable;
2962 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2963 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2964 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2965 if (!mgr || !mgr->IsInitialized()) {
2966 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2967 return;
2968 }
2969 // Check analysis type
2970 TObject *handler;
2971 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2972 handler = (TObject*)mgr->GetInputEventHandler();
2973 if (handler) {
2974 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2975 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2976 }
2977 TDirectory *cdir = gDirectory;
f10e8481 2978 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2979 if (file) {
e16a394c 2980 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2981 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2982 // Unless merging makes no sense
2983 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2984 mgr->Write();
2985 delete file;
fe2d7fc2 2986 // Enable termination for local jobs
2987 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2988 }
2989 if (cdir) cdir->cd();
f10e8481 2990 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2991 }
2992 Bool_t copy = kTRUE;
d3b18c4c 2993 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 2994 if (copy) {
2995 CdWork();
2996 TString workdir = gGrid->GetHomeDirectory();
2997 workdir += fGridWorkingDir;
d3b18c4c 2998 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 2999 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3000 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3001 }
3002}
3003
3004//______________________________________________________________________________
3005void AliAnalysisAlien::WriteAnalysisMacro()
3006{
3007// Write the analysis macro that will steer the analysis in grid mode.
3008 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3009 ofstream out;
3010 out.open(fAnalysisMacro.Data(), ios::out);
3011 if (!out.good()) {
3012 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3013 return;
3014 }
5b273635 3015 Bool_t hasSTEERBase = kFALSE;
3016 Bool_t hasESD = kFALSE;
3017 Bool_t hasAOD = kFALSE;
3018 Bool_t hasANALYSIS = kFALSE;
3019 Bool_t hasANALYSISalice = kFALSE;
3020 Bool_t hasCORRFW = kFALSE;
c57f56b7 3021 TString func = fAnalysisMacro;
3022 TString type = "ESD";
3023 TString comment = "// Analysis using ";
3024 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3025 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3026 type = "AOD";
3027 comment += "AOD";
3028 }
0df6ccf2 3029 if (type!="AOD" && fFriendChainName!="") {
3030 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3031 return;
3032 }
c57f56b7 3033 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3034 else comment += " data";
3035 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3036 func.ReplaceAll(".C", "");
3037 out << "void " << func.Data() << "()" << endl;
3038 out << "{" << endl;
3039 out << comment.Data() << endl;
3040 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3041 out << " TStopwatch timer;" << endl;
3042 out << " timer.Start();" << endl << endl;
13f0c00b 3043 // Change temp directory to current one
3044 out << "// Set temporary merging directory to current one" << endl;
3045 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
b3e07543 3046 // Reset existing include path
3047 out << "// Reset existing include path and add current directory first in the search" << endl;
3048 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3049 if (!fExecutableCommand.Contains("aliroot")) {
3050 out << "// load base root libraries" << endl;
3051 out << " gSystem->Load(\"libTree\");" << endl;
3052 out << " gSystem->Load(\"libGeom\");" << endl;
3053 out << " gSystem->Load(\"libVMC\");" << endl;
3054 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3055 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3056 }
d5c6455a 3057 if (fAdditionalRootLibs.Length()) {
47a4137d 3058 // in principle libtree /lib geom libvmc etc. can go into this list, too
3059 out << "// Add aditional libraries" << endl;
3060 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3061 TIter next(list);
3062 TObjString *str;
3063 while((str=(TObjString*)next())) {
3064 if (str->GetString().Contains(".so"))
3065 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3066 }
47a4137d 3067 if (list) delete list;
d5c6455a 3068 }
57377eb5 3069 out << "// Load analysis framework libraries" << endl;
652561d5 3070 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3071 if (!fPackages) {
f790bc1b 3072 if (!fExecutableCommand.Contains("aliroot")) {
3073 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3074 out << " gSystem->Load(\"libESD\");" << endl;
3075 out << " gSystem->Load(\"libAOD\");" << endl;
3076 }
4e5c5506 3077 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3078 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3079 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3080 } else {
4e5c5506 3081 TIter next(fPackages);
3082 TObject *obj;
57377eb5 3083 TString pkgname;
57377eb5 3084 while ((obj=next())) {
3085 pkgname = obj->GetName();
4478e6f1 3086 if (pkgname == "STEERBase" ||
3087 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3088 if (pkgname == "ESD" ||
3089 pkgname == "ESD.par") hasESD = kTRUE;
3090 if (pkgname == "AOD" ||
3091 pkgname == "AOD.par") hasAOD = kTRUE;
3092 if (pkgname == "ANALYSIS" ||
3093 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3094 if (pkgname == "ANALYSISalice" ||
3095 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3096 if (pkgname == "CORRFW" ||
3097 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3098 }
3099 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3100 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3101 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3102 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3103 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3104 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3105 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3106 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3107 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3108 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3109 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3110 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3111 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3112 out << "// Compile other par packages" << endl;
3113 next.Reset();
fcc9bb6f 3114 while ((obj=next())) {
3115 pkgname = obj->GetName();
4478e6f1 3116 if (pkgname == "STEERBase" ||
3117 pkgname == "STEERBase.par" ||
3118 pkgname == "ESD" ||
3119 pkgname == "ESD.par" ||
3120 pkgname == "AOD" ||
3121 pkgname == "AOD.par" ||
3122 pkgname == "ANALYSIS" ||
3123 pkgname == "ANALYSIS.par" ||
3124 pkgname == "ANALYSISalice" ||
3125 pkgname == "ANALYSISalice.par" ||
3126 pkgname == "CORRFW" ||
3127 pkgname == "CORRFW.par") continue;
5b273635 3128 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3129 }
4e5c5506 3130 }
b3e07543 3131 out << "// include path" << endl;
3132 // Get the include path from the interpreter and remove entries pointing to AliRoot
3133 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3134 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3135 out << " TIter nextpath(listpaths);" << endl;
3136 out << " TObjString *pname;" << endl;
3137 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3138 out << " TString current = pname->GetName();" << endl;
3139 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3140 out << " gSystem->AddIncludePath(current);" << endl;
3141 out << " }" << endl;
3142 out << " if (listpaths) delete listpaths;" << endl;
3143 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3144 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3145 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3146 if (fAdditionalLibs.Length()) {
3147 out << "// Add aditional AliRoot libraries" << endl;
3148 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3149 TIter next(list);
3150 TObjString *str;
3151 while((str=(TObjString*)next())) {
3152 if (str->GetString().Contains(".so"))
3153 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3154 if (str->GetString().Contains(".par"))
3155 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3156 }
3157 if (list) delete list;
3158 }
3159 out << endl;
c57f56b7 3160 out << "// analysis source to be compiled at runtime (if any)" << endl;
3161 if (fAnalysisSource.Length()) {
3162 TObjArray *list = fAnalysisSource.Tokenize(" ");
3163 TIter next(list);
3164 TObjString *str;
3165 while((str=(TObjString*)next())) {
3166 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3167 }
3168 if (list) delete list;
3169 }
3170 out << endl;
b3e07543 3171// out << " printf(\"Currently load libraries:\\n\");" << endl;
3172// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3173 if (fFastReadOption) {
d3339be3 3174 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3175 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3176 out << "// fast xrootd reading enabled" << endl;
3177 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3178 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3179 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3180 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3181 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3182 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3183 }
c57f56b7 3184 out << "// connect to AliEn and make the chain" << endl;
3185 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3186 out << "// read the analysis manager from file" << endl;
f10e8481 3187 TString analysisFile = fExecutable;
3188 analysisFile.ReplaceAll(".sh", ".root");
3189 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3190 out << " if (!file) return;" << endl;
3191 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3192 out << " AliAnalysisManager *mgr = 0;" << endl;
3193 out << " TKey *key;" << endl;
3194 out << " while ((key=(TKey*)nextkey())) {" << endl;
3195 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3196 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3197 out << " };" << endl;
3198 out << " if (!mgr) {" << endl;
205b201f 3199 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3200 out << " return;" << endl;
3201 out << " }" << endl << endl;
3202 out << " mgr->PrintStatus();" << endl;
52b6a92b 3203 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3204 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3205 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3206 } else {
9a8288ff 3207 if (TestBit(AliAnalysisGrid::kTest))
3208 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3209 else
3210 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3211 }
3212 }
242accb2 3213 if (IsUsingTags()) {
3214 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3215 } else {
3216 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3217 }
c57f56b7 3218 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3219 out << " timer.Stop();" << endl;
3220 out << " timer.Print();" << endl;
c57f56b7 3221 out << "}" << endl << endl;
3222 if (IsUsingTags()) {
3223 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3224 out << "{" << endl;
3225 out << "// Create a chain using tags from the xml file." << endl;
3226 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3227 out << " if (!coll) {" << endl;
3228 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3229 out << " return NULL;" << endl;
3230 out << " }" << endl;
3231 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3232 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3233 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3234 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3235 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3236 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3237 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3238 out << " // Check if the cuts configuration file was provided" << endl;
3239 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3240 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3241 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3242 out << " }" << endl;
0df6ccf2 3243 if (fFriendChainName=="") {
3244 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3245 } else {
3246 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3247 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3248 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3249 }
c57f56b7 3250 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3251 out << " chain->ls();" << endl;
3252 out << " return chain;" << endl;
fcc9bb6f 3253 out << "}" << endl << endl;
c57f56b7 3254 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3255 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3256 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3257 msg += " AliLHCTagCuts *lhcCuts,\n";
3258 msg += " AliDetectorTagCuts *detCuts,\n";
3259 msg += " AliEventTagCuts *evCuts)";
23329835 3260 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3261 }
0df6ccf2 3262 }
3263 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3264 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3265 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3266 out << "{" << endl;
3267 out << "// Create a chain using url's from xml file" << endl;
242accb2 3268 out << " TString filename;" << endl;
3269 out << " Int_t run = 0;" << endl;
c57f56b7 3270 out << " TString treename = type;" << endl;
3271 out << " treename.ToLower();" << endl;
3272 out << " treename += \"Tree\";" << endl;
e02fee64 3273 out << " printf(\"***************************************\\n\");" << endl;
3274 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3275 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3276 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3277 out << " if (!coll) {" << endl;
3278 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3279 out << " return NULL;" << endl;
3280 out << " }" << endl;
242accb2 3281 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3282 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3283 if(fFriendChainName!="") {
3284 out << " TChain *chainFriend = new TChain(treename);" << endl;
3285 }
c57f56b7 3286 out << " coll->Reset();" << endl;
0df6ccf2 3287 out << " while (coll->Next()) {" << endl;
242accb2 3288 out << " filename = coll->GetTURL("");" << endl;
3289 out << " if (mgr) {" << endl;
3290 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3291 out << " if (nrun && nrun != run) {" << endl;
3292 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3293 out << " mgr->SetRunFromPath(nrun);" << endl;
3294 out << " run = nrun;" << endl;
3295 out << " }" << endl;
3296 out << " }" << endl;
3297 out << " chain->Add(filename);" << endl;
0df6ccf2 3298 if(fFriendChainName!="") {
3299 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3300 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3301 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3302 out << " chainFriend->Add(fileFriend.Data());" << endl;
3303 }
3304 out << " }" << endl;
c57f56b7 3305 out << " if (!chain->GetNtrees()) {" << endl;
3306 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3307 out << " return NULL;" << endl;
3308 out << " }" << endl;
0df6ccf2 3309 if(fFriendChainName!="") {
3310 out << " chain->AddFriend(chainFriend);" << endl;
3311 }
c57f56b7 3312 out << " return chain;" << endl;
fcc9bb6f 3313 out << "}" << endl << endl;
c57f56b7 3314 }
5b273635 3315 if (hasANALYSISalice) {
3316 out <<"//________________________________________________________________________________" << endl;
3317 out << "Bool_t SetupPar(const char *package) {" << endl;
3318 out << "// Compile the package and set it up." << endl;
3319 out << " TString pkgdir = package;" << endl;
3320 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3321 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3322 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3323 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3324 out << " // Check for BUILD.sh and execute" << endl;
3325 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3326 out << " printf(\"*******************************\\n\");" << endl;
3327 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3328 out << " printf(\"*******************************\\n\");" << endl;
3329 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3330 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3331 out << " gSystem->ChangeDirectory(cdir);" << endl;
3332 out << " return kFALSE;" << endl;
3333 out << " }" << endl;
3334 out << " } else {" << endl;
3335 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3336 out << " gSystem->ChangeDirectory(cdir);" << endl;
3337 out << " return kFALSE;" << endl;
3338 out << " }" << endl;
3339 out << " // Check for SETUP.C and execute" << endl;
3340 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3341 out << " printf(\"*******************************\\n\");" << endl;
3342 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3343 out << " printf(\"*******************************\\n\");" << endl;
3344 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3345 out << " } else {" << endl;
3346 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3347 out << " gSystem->ChangeDirectory(cdir);" << endl;
3348 out << " return kFALSE;" << endl;
3349 out << " }" << endl;
3350 out << " // Restore original workdir" << endl;
3351 out << " gSystem->ChangeDirectory(cdir);" << endl;
3352 out << " return kTRUE;" << endl;
3353 out << "}" << endl;
3354 }
c57f56b7 3355 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3356 }
3357 Bool_t copy = kTRUE;
d3b18c4c 3358 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3359 if (copy) {
3360 CdWork();
3361 TString workdir = gGrid->GetHomeDirectory();
3362 workdir += fGridWorkingDir;
3363 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3364 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3365 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3366 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3367 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3368 }
3369 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3370 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3371 }
3372}
3373
0f389141 3374//______________________________________________________________________________
3375void AliAnalysisAlien::WriteMergingMacro()
3376{
3377// Write a macro to merge the outputs per master job.
3378 if (!fMergeViaJDL) return;
3379 if (!fOutputFiles.Length()) {
3380 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3381 return;
3382 }
3383 TString mergingMacro = fExecutable;
3384 mergingMacro.ReplaceAll(".sh","_merge.C");
3385 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3386 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3387 ofstream out;
3388 out.open(mergingMacro.Data(), ios::out);
3389 if (!out.good()) {
3390 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3391 return;
3392 }
5b273635 3393 Bool_t hasSTEERBase = kFALSE;
3394 Bool_t hasESD = kFALSE;
3395 Bool_t hasAOD = kFALSE;
3396 Bool_t hasANALYSIS = kFALSE;
3397 Bool_t hasANALYSISalice = kFALSE;
3398 Bool_t hasCORRFW = kFALSE;
0f389141 3399 TString func = mergingMacro;
3400 TString comment;
3401 func.ReplaceAll(".C", "");
f47d5cb4 3402 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3403 out << "{" << endl;
3404 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3405 out << " TStopwatch timer;" << endl;
3406 out << " timer.Start();" << endl << endl;
b3e07543 3407 // Reset existing include path
3408 out << "// Reset existing include path and add current directory first in the search" << endl;
3409 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3410 if (!fExecutableCommand.Contains("aliroot")) {
3411 out << "// load base root libraries" << endl;
3412 out << " gSystem->Load(\"libTree\");" << endl;
3413 out << " gSystem->Load(\"libGeom\");" << endl;
3414 out << " gSystem->Load(\"libVMC\");" << endl;
3415 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3416 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3417 }
0f389141 3418 if (fAdditionalRootLibs.Length()) {
3419 // in principle libtree /lib geom libvmc etc. can go into this list, too
3420 out << "// Add aditional libraries" << endl;
3421 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3422 TIter next(list);
3423 TObjString *str;
3424 while((str=(TObjString*)next())) {
3425 if (str->GetString().Contains(".so"))
3426 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3427 }
3428 if (list) delete list;
3429 }
0f389141 3430 out << "// Load analysis framework libraries" << endl;
3431 if (!fPackages) {
7c2cd90a 3432 if (!fExecutableCommand.Contains("aliroot")) {
3433 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3434 out << " gSystem->Load(\"libESD\");" << endl;
3435 out << " gSystem->Load(\"libAOD\");" << endl;
3436 }
0f389141 3437 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3438 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3439 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3440 } else {
3441 TIter next(fPackages);
3442 TObject *obj;
3443 TString pkgname;
5b273635 3444 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3445 while ((obj=next())) {
3446 pkgname = obj->GetName();
3447 if (pkgname == "STEERBase" ||
3448 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3449 if (pkgname == "ESD" ||
3450 pkgname == "ESD.par") hasESD = kTRUE;
3451 if (pkgname == "AOD" ||
3452 pkgname == "AOD.par") hasAOD = kTRUE;
3453 if (pkgname == "ANALYSIS" ||
3454 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3455 if (pkgname == "ANALYSISalice" ||
3456 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3457 if (pkgname == "CORRFW" ||
3458 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3459 }
5b273635 3460 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3461 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3462 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3463 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3464 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3465 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3466 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3467 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3468 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3469 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3470 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3471 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3472 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3473 out << "// Compile other par packages" << endl;
3474 next.Reset();
3475 while ((obj=next())) {
3476 pkgname = obj->GetName();
3477 if (pkgname == "STEERBase" ||
3478 pkgname == "STEERBase.par" ||
3479 pkgname == "ESD" ||
3480 pkgname == "ESD.par" ||
3481 pkgname == "AOD" ||
3482 pkgname == "AOD.par" ||
3483 pkgname == "ANALYSIS" ||
3484 pkgname == "ANALYSIS.par" ||
3485 pkgname == "ANALYSISalice" ||
3486 pkgname == "ANALYSISalice.par" ||
3487 pkgname == "CORRFW" ||
3488 pkgname == "CORRFW.par") continue;
5b273635 3489 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3490 }
3491 }
b3e07543 3492 out << "// include path" << endl;
3493 // Get the include path from the interpreter and remove entries pointing to AliRoot
3494 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3495 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3496 out << " TIter nextpath(listpaths);" << endl;
3497 out << " TObjString *pname;" << endl;
3498 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3499 out << " TString current = pname->GetName();" << endl;
3500 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3501 out << " gSystem->AddIncludePath(current);" << endl;
3502 out << " }" << endl;
3503 out << " if (listpaths) delete listpaths;" << endl;
3504 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3505 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3506 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3507 if (fAdditionalLibs.Length()) {
3508 out << "// Add aditional AliRoot libraries" << endl;
3509 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3510 TIter next(list);
3511 TObjString *str;
3512 while((str=(TObjString*)next())) {
3513 if (str->GetString().Contains(".so"))
3514 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3515 }
3516 if (list) delete list;
3517 }
3518 out << endl;
3519 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3520 if (fAnalysisSource.Length()) {
3521 TObjArray *list = fAnalysisSource.Tokenize(" ");
3522 TIter next(list);
3523 TObjString *str;
3524 while((str=(TObjString*)next())) {
3525 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3526 }
3527 if (list) delete list;
3528 }
149d288c 3529 out << endl;
3530
0f389141 3531 if (fFastReadOption) {
3532 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3533 out << "// fast xrootd reading enabled" << endl;
3534 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3535 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3536 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3537 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3538 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3539 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3540 }
3541 // Change temp directory to current one
3542 out << "// Set temporary merging directory to current one" << endl;
3543 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3544 out << "// Connect to AliEn" << endl;
3545 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3546 out << " TString outputDir = dir;" << endl;
f790bc1b 3547 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3548 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3549 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3550 out << " TIter *iter = new TIter(list);" << endl;
3551 out << " TObjString *str;" << endl;
a2f5fc01 3552 out << " TString outputFile;" << endl;
0f389141 3553 out << " Bool_t merged = kTRUE;" << endl;
3554 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3555 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3556 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3557 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3558 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3559 out << " // Skip already merged outputs" << endl;
a2f5fc01 3560 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3561 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3562 out << " continue;" << endl;
3563 out << " }" << endl;
a2f5fc01 3564 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3565 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3566 out << " if (!merged) {" << endl;
a2f5fc01 3567 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3568 out << " return;" << endl;
0f389141 3569 out << " }" << endl;
3570 out << " }" << endl;
7c2cd90a 3571 out << " // all outputs merged, validate" << endl;
3572 out << " ofstream out;" << endl;
3573 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3574 out << " out.close();" << endl;
3575 out << " // read the analysis manager from file" << endl;
f866cba5 3576 TString analysisFile = fExecutable;
3577 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3578 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3579 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3580 out << " if (!file) return;" << endl;
f866cba5 3581 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3582 out << " AliAnalysisManager *mgr = 0;" << endl;
3583 out << " TKey *key;" << endl;
3584 out << " while ((key=(TKey*)nextkey())) {" << endl;
3585 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3586 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3587 out << " };" << endl;
3588 out << " if (!mgr) {" << endl;
3589 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3590 out << " return;" << endl;
3591 out << " }" << endl << endl;
5c4250fc 3592 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3593 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3594 out << " mgr->PrintStatus();" << endl;
3595 if (AliAnalysisManager::GetAnalysisManager()) {
3596 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3597 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3598 } else {
9a8288ff 3599 if (TestBit(AliAnalysisGrid::kTest))
3600 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3601 else
3602 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3603 }
3604 }
9a8288ff 3605 out << " TTree *tree = NULL;" << endl;
3606 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3607 out << "}" << endl << endl;
5b273635 3608 if (hasANALYSISalice) {
3609 out <<"//________________________________________________________________________________" << endl;
3610 out << "Bool_t SetupPar(const char *package) {" << endl;
3611 out << "// Compile the package and set it up." << endl;
3612 out << " TString pkgdir = package;" << endl;
3613 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3614 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3615 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3616 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3617 out << " // Check for BUILD.sh and execute" << endl;
3618 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3619 out << " printf(\"*******************************\\n\");" << endl;
3620 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3621 out << " printf(\"*******************************\\n\");" << endl;
3622 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3623 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3624 out << " gSystem->ChangeDirectory(cdir);" << endl;
3625 out << " return kFALSE;" << endl;
3626 out << " }" << endl;
3627 out << " } else {" << endl;
3628 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3629 out << " gSystem->ChangeDirectory(cdir);" << endl;
3630 out << " return kFALSE;" << endl;
3631 out << " }" << endl;
3632 out << " // Check for SETUP.C and execute" << endl;
3633 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3634 out << " printf(\"*******************************\\n\");" << endl;
3635 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3636 out << " printf(\"*******************************\\n\");" << endl;
3637 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3638 out << " } else {" << endl;
3639 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3640 out << " gSystem->ChangeDirectory(cdir);" << endl;
3641 out << " return kFALSE;" << endl;
3642 out << " }" << endl;
3643 out << " // Restore original workdir" << endl;
3644 out << " gSystem->ChangeDirectory(cdir);" << endl;
3645 out << " return kTRUE;" << endl;
3646 out << "}" << endl;
3647 }
0f389141 3648 }
3649 Bool_t copy = kTRUE;
d3b18c4c 3650 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3651 if (copy) {
3652 CdWork();
3653 TString workdir = gGrid->GetHomeDirectory();
3654 workdir += fGridWorkingDir;
3655 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3656 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3657 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3658 }
3659}
3660
3661//______________________________________________________________________________
3662Bool_t AliAnalysisAlien::SetupPar(const char *package)
3663{
205b201f 3664// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3665// Note that for loading the compiled library. The current directory should have precedence in
3666// LD_LIBRARY_PATH
3667 TString pkgdir = package;
3668 pkgdir.ReplaceAll(".par","");
b50a41c3 3669 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3670 TString cdir = gSystem->WorkingDirectory();
3671 gSystem->ChangeDirectory(pkgdir);
3672 // Check for BUILD.sh and execute
3673 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3674 printf("**************************************************\n");
3675 printf("*** Building PAR archive %s\n", package);
3676 printf("**************************************************\n");
3677 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3678 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3679 gSystem->ChangeDirectory(cdir);
3680 return kFALSE;
3681 }
3682 } else {
3683 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3684 gSystem->ChangeDirectory(cdir);
3685 return kFALSE;
3686 }
3687 // Check for SETUP.C and execute
3688 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3689 printf("**************************************************\n");
3690 printf("*** Setup PAR archive %s\n", package);
3691 printf("**************************************************\n");
3692 gROOT->Macro("PROOF-INF/SETUP.C");
3693 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3694 } else {
3695 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3696 gSystem->ChangeDirectory(cdir);
3697 return kFALSE;
3698 }
3699 // Restore original workdir
3700 gSystem->ChangeDirectory(cdir);
3701 return kTRUE;
3702}
3703
c57f56b7 3704//______________________________________________________________________________
3705void AliAnalysisAlien::WriteExecutable()
3706{
3707// Generate the alien executable script.
3708 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3709 ofstream out;
3710 out.open(fExecutable.Data(), ios::out);
3711 if (out.bad()) {
5513444a 3712 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3713 return;
3714 }
3715 out << "#!/bin/bash" << endl;
2a4d5166 3716 // Make sure we can properly compile par files
3717 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3718 out << "echo \"=========================================\"" << endl;
3719 out << "echo \"############## PATH : ##############\"" << endl;
3720 out << "echo $PATH" << endl;
3721 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3722 out << "echo $LD_LIBRARY_PATH" << endl;
3723 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3724 out << "echo $ROOTSYS" << endl;
3725 out << "echo \"############## which root : ##############\"" << endl;
3726 out << "which root" << endl;
3727 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3728 out << "echo $ALICE_ROOT" << endl;
3729 out << "echo \"############## which aliroot : ##############\"" << endl;
3730 out << "which aliroot" << endl;
9c5ddadc 3731 out << "echo \"############## system limits : ##############\"" << endl;
3732 out << "ulimit -a" << endl;
3733 out << "echo \"############## memory : ##############\"" << endl;
3734 out << "free -m" << endl;
c57f56b7 3735 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3736 out << fExecutableCommand << " ";
631c0b05 3737 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3738 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3739 out << "echo \"############## memory after: ##############\"" << endl;
3740 out << "free -m" << endl;
c57f56b7 3741 }
3742 Bool_t copy = kTRUE;
d3b18c4c 3743 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3744 if (copy) {
3745 CdWork();
3746 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3747 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3748 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3749 workdir += fGridWorkingDir;
3750 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3751 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3752 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3753 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3754 }
3755}
3756
0f389141 3757//______________________________________________________________________________
3758void AliAnalysisAlien::WriteMergeExecutable()
3759{
3760// Generate the alien executable script for the merging job.
3761 if (!fMergeViaJDL) return;
3762 TString mergeExec = fExecutable;
3763 mergeExec.ReplaceAll(".sh", "_merge.sh");
3764 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3765 ofstream out;
3766 out.open(mergeExec.Data(), ios::out);
3767 if (out.bad()) {
3768 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3769 return;
3770 }
3771 out << "#!/bin/bash" << endl;
2a4d5166 3772 // Make sure we can properly compile par files
3773 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3774 out << "echo \"=========================================\"" << endl;
3775 out << "echo \"############## PATH : ##############\"" << endl;
3776 out << "echo $PATH" << endl;
3777 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3778 out << "echo $LD_LIBRARY_PATH" << endl;
3779 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3780 out << "echo $ROOTSYS" << endl;
3781 out << "echo \"############## which root : ##############\"" << endl;
3782 out << "which root" << endl;
3783 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3784 out << "echo $ALICE_ROOT" << endl;
3785 out << "echo \"############## which aliroot : ##############\"" << endl;
3786 out << "which aliroot" << endl;
3787 out << "echo \"############## system limits : ##############\"" << endl;
3788 out << "ulimit -a" << endl;
3789 out << "echo \"############## memory : ##############\"" << endl;
3790 out << "free -m" << endl;
3791 out << "echo \"=========================================\"" << endl << endl;
0f389141 3792 TString mergeMacro = fExecutable;
3793 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3794 if (IsOneStageMerging())
3795 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3796 else
f47d5cb4 3797 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3798 out << fExecutableCommand << " " << "$ARG" << endl;
3799 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3800 out << "echo \"############## memory after: ##############\"" << endl;
3801 out << "free -m" << endl;
0f389141 3802 }
3803 Bool_t copy = kTRUE;
d3b18c4c 3804 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3805 if (copy) {
3806 CdWork();
3807 TString workdir = gGrid->GetHomeDirectory();
3808 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3809 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3810 workdir += fGridWorkingDir;
3811 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3812 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3813 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3814 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3815 }
3816}
3817
c57f56b7 3818//______________________________________________________________________________
5513444a 3819void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3820{
3821// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3822// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3823// Next lines: full_path_to_dataset XXX (XXX is a string)
3824// To submit, one has to: submit jdl XXX for all lines
3825 ofstream out;
3826 out.open(filename, ios::out);
3827 if (out.bad()) {
3828 Error("WriteProductionFile", "Bad file name: %s", filename);
3829 return;
3830 }
d3b18c4c 3831 TString workdir;
3832 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3833 workdir = gGrid->GetHomeDirectory();
5513444a 3834 workdir += fGridWorkingDir;
f5e8c702 3835 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3836 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3837 out << locjdl << " " << njobspermaster << endl;
5513444a 3838 Int_t nmasterjobs = fInputFiles->GetEntries();
3839 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3840 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3841 runOutDir.ReplaceAll(".xml", "");
3842 if (fOutputToRunNo)
3843 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3844 else
3845 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3846 }
d3b18c4c 3847 if (gGrid) {
3848 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3849 if (FileExists(filename)) gGrid->Rm(filename);
3850 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3851 }
5513444a 3852}
3853
3854//______________________________________________________________________________
0f389141 3855void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3856{
3857// Generate the alien validation script.
3858 // Generate the validation script
3859 TObjString *os;
d3b18c4c 3860 if (fValidationScript.IsNull()) {
3861 fValidationScript = fExecutable;
3862 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3863 }
3864 TString validationScript = fValidationScript;
3865 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3866 if (!Connect()) {
3867 Error("WriteValidationScript", "Alien connection required");
3868 return;
3869 }
d3b18c4c 3870 if (!fTerminateFiles.IsNull()) {
3871 fTerminateFiles.Strip();
3872 fTerminateFiles.ReplaceAll(" ",",");
3873 }
a2f5fc01 3874 TString outStream = "";
3875 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3876 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3877 ofstream out;
0d5d317c 3878 out.open(validationScript, ios::out);
c57f56b7 3879 out << "#!/bin/bash" << endl;
3880 out << "##################################################" << endl;
3881 out << "validateout=`dirname $0`" << endl;
3882 out << "validatetime=`date`" << endl;
3883 out << "validated=\"0\";" << endl;
3884 out << "error=0" << endl;
3885 out << "if [ -z $validateout ]" << endl;
3886 out << "then" << endl;
3887 out << " validateout=\".\"" << endl;
3888 out << "fi" << endl << endl;
3889 out << "cd $validateout;" << endl;
3890 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3891 out << "echo \"*******************************************************\"" << outStream << endl;
3892 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3893 out << "" << endl;
a2f5fc01 3894 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3895 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3896 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3897 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3898 out << "ls -la ./" << outStream << endl;
3899 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3900 out << "##################################################" << endl;
ebec370a 3901 out << "" << endl;
3902
3903 out << "if [ ! -f stderr ] ; then" << endl;
3904 out << " error=1" << endl;
a2f5fc01 3905 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3906 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3907 out << "fi" << endl;
3908
b34c9f51 3909 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3910 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3911 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3912 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3913 out << "" << endl;
3914
ebec370a 3915 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3916 out << " error=1" << endl;
a2f5fc01 3917 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3918 out << " echo \"$parArch\" " << outStream << endl;
3919 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3920 out << "fi" << endl;
3921
3922 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3923 out << " error=1" << endl;
a2f5fc01 3924 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3925 out << " echo \"$segViol\" " << outStream << endl;
3926 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3927 out << "fi" << endl;
3928
3929 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3930 out << " error=1" << endl;
a2f5fc01 3931 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3932 out << " echo \"$segFault\" " << outStream << endl;
3933 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3934 out << "fi" << endl;
3935
b34c9f51 3936 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3937 out << " error=1" << endl;
a2f5fc01 3938 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3939 out << " echo \"$glibcErr\" " << outStream << endl;
3940 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3941 out << "fi" << endl;
3942
ebec370a 3943 // Part dedicated to the specific analyses running into the train
3944
d3b18c4c 3945 TString outputFiles = fOutputFiles;
3946 if (merge && !fTerminateFiles.IsNull()) {
3947 outputFiles += ",";
3948 outputFiles += fTerminateFiles;
3949 }
3950 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3951 TIter next1(arr);
a2f5fc01 3952 TString outputFile;
bbd88db8 3953 while (!merge && (os=(TObjString*)next1())) {
3954 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3955 outputFile = os->GetString();
3956 Int_t index = outputFile.Index("@");
3957 if (index > 0) outputFile.Remove(index);
bbd88db8 3958 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3959 if (outputFile.Contains("*")) continue;
3960 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3961 out << " error=1" << endl;
7c2cd90a 3962 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3963 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3964 out << "fi" << endl;
3965 }
3966 delete arr;
7c2cd90a 3967 out << "if ! [ -f outputs_valid ] ; then" << endl;
3968 out << " error=1" << endl;
3969 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3970 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3971 out << "fi" << endl;
923e2ca5 3972
c57f56b7 3973 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3974 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3975 if (!IsKeepLogs()) {
3976 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3977 outStream = "";
149d288c 3978 out << " rm -f std*" << endl;
3979 }
c57f56b7 3980 out << "fi" << endl;
3981
a2f5fc01 3982 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3983 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3984 out << "cd -" << endl;
3985 out << "exit $error" << endl;
3986 }
3987 Bool_t copy = kTRUE;
d3b18c4c 3988 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3989 if (copy) {
3990 CdWork();
3991 TString workdir = gGrid->GetHomeDirectory();
3992 workdir += fGridWorkingDir;
d3b18c4c 3993 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 3994 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3995 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3996 }
3997}