]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Clean way to get input from clusterizer and event
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
b3e07543 61 fMaxMergeStages(0),
d2a409b2 62 fNsubmitted(0),
a3e84053 63 fProductionMode(0),
cd11251e 64 fOutputToRunNo(0),
0f389141 65 fMergeViaJDL(0),
66 fFastReadOption(0),
e1c22e21 67 fOverwriteMode(1),
149d288c 68 fNreplicas(2),
3bdcb562 69 fNproofWorkers(0),
27734f0e 70 fNproofWorkersPerSlave(0),
3bdcb562 71 fProofReset(0),
c57f56b7 72 fRunNumbers(),
73 fExecutable(),
0a1c1f7f 74 fExecutableCommand(),
c57f56b7 75 fArguments(),
631c0b05 76 fExecutableArgs(),
c57f56b7 77 fAnalysisMacro(),
78 fAnalysisSource(),
d3b18c4c 79 fValidationScript(),
d5c6455a 80 fAdditionalRootLibs(),
c57f56b7 81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
648174cf 86 fExternalPackages(),
c57f56b7 87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
e7c71df0 95 fDatasetName(),
c57f56b7 96 fJDLName(),
d3b18c4c 97 fTerminateFiles(),
bb885a9e 98 fMergeExcludes(),
f965131e 99 fIncludePath(),
bb885a9e 100 fCloseSE(),
0df6ccf2 101 fFriendChainName(),
c6cb3634 102 fJobTag(),
648174cf 103 fOutputSingle(),
5fce53f4 104 fRunPrefix(),
3bdcb562 105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
f47d5cb4 110 fMergeDirName(),
4e5c5506 111 fInputFiles(0),
112 fPackages(0)
c57f56b7 113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
0f389141 122 fMergingJDL(NULL),
c57f56b7 123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
bb885a9e 128 fNtestFiles(0),
319593fb 129 fNrunsPerMaster(0),
16a4353c 130 fMaxMergeFiles(0),
b3e07543 131 fMaxMergeStages(0),
d2a409b2 132 fNsubmitted(0),
a3e84053 133 fProductionMode(0),
cd11251e 134 fOutputToRunNo(0),
0f389141 135 fMergeViaJDL(0),
136 fFastReadOption(0),
e1c22e21 137 fOverwriteMode(1),
149d288c 138 fNreplicas(2),
3bdcb562 139 fNproofWorkers(0),
27734f0e 140 fNproofWorkersPerSlave(0),
3bdcb562 141 fProofReset(0),
c57f56b7 142 fRunNumbers(),
143 fExecutable(),
0a1c1f7f 144 fExecutableCommand(),
c57f56b7 145 fArguments(),
631c0b05 146 fExecutableArgs(),
c57f56b7 147 fAnalysisMacro(),
148 fAnalysisSource(),
d3b18c4c 149 fValidationScript(),
d5c6455a 150 fAdditionalRootLibs(),
c57f56b7 151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
648174cf 156 fExternalPackages(),
c57f56b7 157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
e7c71df0 165 fDatasetName(),
c57f56b7 166 fJDLName(),
d3b18c4c 167 fTerminateFiles(),
bb885a9e 168 fMergeExcludes(),
f965131e 169 fIncludePath(),
bb885a9e 170 fCloseSE(),
0df6ccf2 171 fFriendChainName(),
c6cb3634 172 fJobTag(),
648174cf 173 fOutputSingle(),
5fce53f4 174 fRunPrefix(),
3bdcb562 175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
f47d5cb4 180 fMergeDirName(),
4e5c5506 181 fInputFiles(0),
182 fPackages(0)
c57f56b7 183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
0f389141 192 fMergingJDL(NULL),
c57f56b7 193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 198 fNtestFiles(other.fNtestFiles),
319593fb 199 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 200 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 201 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 202 fNsubmitted(other.fNsubmitted),
a3e84053 203 fProductionMode(other.fProductionMode),
cd11251e 204 fOutputToRunNo(other.fOutputToRunNo),
0f389141 205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
149d288c 208 fNreplicas(other.fNreplicas),
3bdcb562 209 fNproofWorkers(other.fNproofWorkers),
27734f0e 210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 211 fProofReset(other.fProofReset),
c57f56b7 212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
0a1c1f7f 214 fExecutableCommand(other.fExecutableCommand),
c57f56b7 215 fArguments(other.fArguments),
631c0b05 216 fExecutableArgs(other.fExecutableArgs),
c57f56b7 217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 219 fValidationScript(other.fValidationScript),
d5c6455a 220 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
648174cf 226 fExternalPackages(other.fExternalPackages),
c57f56b7 227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
e7c71df0 235 fDatasetName(other.fDatasetName),
c57f56b7 236 fJDLName(other.fJDLName),
d3b18c4c 237 fTerminateFiles(other.fTerminateFiles),
bb885a9e 238 fMergeExcludes(other.fMergeExcludes),
f965131e 239 fIncludePath(other.fIncludePath),
bb885a9e 240 fCloseSE(other.fCloseSE),
0df6ccf2 241 fFriendChainName(other.fFriendChainName),
c6cb3634 242 fJobTag(other.fJobTag),
648174cf 243 fOutputSingle(other.fOutputSingle),
5fce53f4 244 fRunPrefix(other.fRunPrefix),
3bdcb562 245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
f47d5cb4 250 fMergeDirName(other.fMergeDirName),
4e5c5506 251 fInputFiles(0),
252 fPackages(0)
c57f56b7 253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
c57f56b7 259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
4e5c5506 266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
c57f56b7 273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
0f389141 280 if (fMergingJDL) delete fMergingJDL;
c57f56b7 281 if (fInputFiles) delete fInputFiles;
4e5c5506 282 if (fPackages) delete fPackages;
c57f56b7 283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 298 fNtestFiles = other.fNtestFiles;
a3e84053 299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 301 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
cd11251e 304 fOutputToRunNo = other.fOutputToRunNo;
0f389141 305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
149d288c 308 fNreplicas = other.fNreplicas;
3bdcb562 309 fNproofWorkers = other.fNproofWorkers;
27734f0e 310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 311 fProofReset = other.fProofReset;
c57f56b7 312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
0a1c1f7f 314 fExecutableCommand = other.fExecutableCommand;
c57f56b7 315 fArguments = other.fArguments;
631c0b05 316 fExecutableArgs = other.fExecutableArgs;
c57f56b7 317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 319 fValidationScript = other.fValidationScript;
d5c6455a 320 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
648174cf 326 fExternalPackages = other.fExternalPackages;
c57f56b7 327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
e7c71df0 335 fDatasetName = other.fDatasetName;
c57f56b7 336 fJDLName = other.fJDLName;
d3b18c4c 337 fTerminateFiles = other.fTerminateFiles;
bb885a9e 338 fMergeExcludes = other.fMergeExcludes;
f965131e 339 fIncludePath = other.fIncludePath;
bb885a9e 340 fCloseSE = other.fCloseSE;
0df6ccf2 341 fFriendChainName = other.fFriendChainName;
c6cb3634 342 fJobTag = other.fJobTag;
648174cf 343 fOutputSingle = other.fOutputSingle;
5fce53f4 344 fRunPrefix = other.fRunPrefix;
3bdcb562 345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
f47d5cb4 350 fMergeDirName = other.fMergeDirName;
c57f56b7 351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
4e5c5506 358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
c57f56b7 365 }
366 return *this;
367}
368
74013535 369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
f965131e 377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 391 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 392}
393
98690895 394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunList(const char* runList)
396{
397// Add several runs into the list of runs; they are expected to be separated by a blank character.
398 TString sList = runList;
399 TObjArray *list = sList.Tokenize(" ");
400 Int_t n = list->GetEntries();
401 for (Int_t i = 0; i < n; i++) {
402 TObjString *os = (TObjString*)list->At(i);
403 AddRunNumber(os->GetString().Atoi());
404 }
405 delete list;
406}
407
ee75cfc3 408//______________________________________________________________________________
409void AliAnalysisAlien::AddRunNumber(const char* run)
410{
411// Add a run number to the list of runs to be processed.
412 if (fRunNumbers.Length()) fRunNumbers += " ";
413 fRunNumbers += run;
414}
415
c57f56b7 416//______________________________________________________________________________
417void AliAnalysisAlien::AddDataFile(const char *lfn)
418{
419// Adds a data file to the input to be analysed. The file should be a valid LFN
420// or point to an existing file in the alien workdir.
421 if (!fInputFiles) fInputFiles = new TObjArray();
422 fInputFiles->Add(new TObjString(lfn));
423}
648174cf 424
425//______________________________________________________________________________
426void AliAnalysisAlien::AddExternalPackage(const char *package)
427{
428// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
429 if (fExternalPackages) fExternalPackages += " ";
430 fExternalPackages += package;
431}
432
c57f56b7 433//______________________________________________________________________________
434Bool_t AliAnalysisAlien::Connect()
435{
436// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
437 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 438 if (fProductionMode) return kTRUE;
c57f56b7 439 if (!gGrid) {
440 Info("Connect", "Trying to connect to AliEn ...");
441 TGrid::Connect("alien://");
442 }
443 if (!gGrid || !gGrid->IsConnected()) {
444 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
445 return kFALSE;
446 }
447 fUser = gGrid->GetUser();
448 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
449 return kTRUE;
450}
451
452//______________________________________________________________________________
453void AliAnalysisAlien::CdWork()
454{
455// Check validity of alien workspace. Create directory if possible.
456 if (!Connect()) {
457 Error("CdWork", "Alien connection required");
458 return;
459 }
460 TString homedir = gGrid->GetHomeDirectory();
461 TString workdir = homedir + fGridWorkingDir;
923e2ca5 462 if (DirectoryExists(workdir)) {
463 gGrid->Cd(workdir);
464 return;
465 }
466 // Work directory not existing - create it
467 gGrid->Cd(homedir);
b93f8109 468 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 469 gGrid->Cd(fGridWorkingDir);
d3b18c4c 470 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 471 } else {
d3b18c4c 472 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 473 workdir.Data(), homedir.Data());
474 fGridWorkingDir = "";
475 }
c57f56b7 476}
477
348be253 478//______________________________________________________________________________
479Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
480{
481// Check if file copying is possible.
d3b18c4c 482 if (fProductionMode) return kTRUE;
348be253 483 if (!Connect()) {
484 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
485 return kFALSE;
486 }
d3339be3 487 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
488 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 489 // Check if alien_CLOSE_SE is defined
490 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
491 if (!closeSE.IsNull()) {
492 Info("CheckFileCopy", "Your current close storage is pointing to: \
493 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
494 } else {
495 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
496 }
497 // Check if grid directory exists.
498 if (!DirectoryExists(alienpath)) {
499 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
500 return kFALSE;
501 }
d3339be3 502 TFile f("plugin_test_copy", "RECREATE");
348be253 503 // User may not have write permissions to current directory
504 if (f.IsZombie()) {
505 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
506 gSystem->WorkingDirectory());
507 return kFALSE;
508 }
509 f.Close();
d3339be3 510 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
511 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
512 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 513 \n# 1. Make sure you have write permissions there. If this is the case: \
514 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
515 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
516 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
517 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
518 gSystem->Unlink(f.GetName());
519 return kFALSE;
520 }
521 gSystem->Unlink(f.GetName());
d3339be3 522 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
523 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 524 return kTRUE;
525}
526
c57f56b7 527//______________________________________________________________________________
528Bool_t AliAnalysisAlien::CheckInputData()
529{
530// Check validity of input data. If necessary, create xml files.
d3b18c4c 531 if (fProductionMode) return kTRUE;
a8739e8a 532 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 533 if (!fGridDataDir.Length()) {
534 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
535 return kFALSE;
536 }
9f020952 537 if (fMergeViaJDL) {
538 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
539 return kFALSE;
540 }
d2a409b2 541 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 542 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
543 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 544 return kTRUE;
c57f56b7 545 }
546 // Process declared files
a2f5fc01 547 Bool_t isCollection = kFALSE;
548 Bool_t isXml = kFALSE;
549 Bool_t useTags = kFALSE;
c57f56b7 550 Bool_t checked = kFALSE;
d3b18c4c 551 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 552 TString file;
553 TString workdir = gGrid->GetHomeDirectory();
554 workdir += fGridWorkingDir;
555 if (fInputFiles) {
556 TObjString *objstr;
557 TIter next(fInputFiles);
558 while ((objstr=(TObjString*)next())) {
559 file = workdir;
560 file += "/";
561 file += objstr->GetString();
562 // Store full lfn path
563 if (FileExists(file)) objstr->SetString(file);
564 else {
565 file = objstr->GetName();
566 if (!FileExists(objstr->GetName())) {
567 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
568 objstr->GetName(), workdir.Data());
569 return kFALSE;
570 }
571 }
572 Bool_t iscoll, isxml, usetags;
573 CheckDataType(file, iscoll, isxml, usetags);
574 if (!checked) {
575 checked = kTRUE;
a2f5fc01 576 isCollection = iscoll;
577 isXml = isxml;
578 useTags = usetags;
579 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 580 } else {
a2f5fc01 581 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 582 Error("CheckInputData", "Some conflict was found in the types of inputs");
583 return kFALSE;
584 }
585 }
586 }
587 }
588 // Process requested run numbers
a8739e8a 589 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 590 // Check validity of alien data directory
591 if (!fGridDataDir.Length()) {
592 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
593 return kFALSE;
594 }
923e2ca5 595 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 596 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
597 return kFALSE;
598 }
a2f5fc01 599 if (isCollection) {
c57f56b7 600 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
601 return kFALSE;
602 }
603
a2f5fc01 604 if (checked && !isXml) {
c57f56b7 605 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
606 return kFALSE;
607 }
608 // Check validity of run number(s)
609 TObjArray *arr;
610 TObjString *os;
74013535 611 TString format;
319593fb 612 Int_t nruns = 0;
904f9f5f 613 TString schunk, schunk2;
c57f56b7 614 TString path;
615 if (!checked) {
616 checked = kTRUE;
a2f5fc01 617 useTags = fDataPattern.Contains("tag");
618 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 619 }
a2f5fc01 620 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 621 Error("CheckInputData", "Cannot mix input files using/not using tags");
622 return kFALSE;
623 }
624 if (fRunNumbers.Length()) {
a8739e8a 625 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 626 arr = fRunNumbers.Tokenize(" ");
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
629 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 630 if (!DirectoryExists(path)) {
631 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 632 continue;
c57f56b7 633 }
634 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
635 TString msg = "\n##### file: ";
636 msg += path;
637 msg += " type: xml_collection;";
a2f5fc01 638 if (useTags) msg += " using_tags: Yes";
c57f56b7 639 else msg += " using_tags: No";
23329835 640 Info("CheckDataType", "%s", msg.Data());
319593fb 641 if (fNrunsPerMaster<2) {
d2a409b2 642 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 643 } else {
644 nruns++;
645 if (((nruns-1)%fNrunsPerMaster) == 0) {
646 schunk = os->GetString();
647 }
648 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
649 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 650 AddDataFile(schunk);
319593fb 651 }
c57f56b7 652 }
653 delete arr;
a8739e8a 654 } else {
655 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
656 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 657 format = Form("%%s/%s ", fRunPrefix.Data());
658 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 659 if (!DirectoryExists(path)) {
a8739e8a 660 continue;
661 }
74013535 662 format = Form("%%s/%s.xml", fRunPrefix.Data());
663 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 664 TString msg = "\n##### file: ";
665 msg += path;
666 msg += " type: xml_collection;";
a2f5fc01 667 if (useTags) msg += " using_tags: Yes";
a8739e8a 668 else msg += " using_tags: No";
23329835 669 Info("CheckDataType", "%s", msg.Data());
319593fb 670 if (fNrunsPerMaster<2) {
74013535 671 format = Form("%s.xml", fRunPrefix.Data());
672 AddDataFile(Form(format.Data(),irun));
319593fb 673 } else {
674 nruns++;
675 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 676 schunk = Form(fRunPrefix.Data(),irun);
319593fb 677 }
74013535 678 format = Form("_%s.xml", fRunPrefix.Data());
679 schunk2 = Form(format.Data(), irun);
319593fb 680 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 681 schunk += schunk2;
d2a409b2 682 AddDataFile(schunk);
319593fb 683 }
a8739e8a 684 }
904f9f5f 685 if (!fInputFiles) {
686 schunk += schunk2;
687 AddDataFile(schunk);
688 }
c57f56b7 689 }
690 return kTRUE;
691}
692
693//______________________________________________________________________________
694Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
695{
696// Create dataset for the grid data directory + run number.
d3b18c4c 697 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 698 if (!Connect()) {
699 Error("CreateDataset", "Cannot create dataset with no grid connection");
700 return kFALSE;
701 }
702
703 // Cd workspace
d3b18c4c 704 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 705 TString workdir = gGrid->GetHomeDirectory();
706 workdir += fGridWorkingDir;
707
708 // Compose the 'find' command arguments
74013535 709 TString format;
c57f56b7 710 TString command;
711 TString options = "-x collection ";
bb885a9e 712 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 713 TString conditions = "";
714
715 TString file;
716 TString path;
319593fb 717 Int_t nruns = 0;
904f9f5f 718 TString schunk, schunk2;
ab254fd1 719 TGridCollection *cbase=0, *cadd=0;
d2a409b2 720 if (!fRunNumbers.Length() && !fRunRange[0]) {
721 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
722 // Make a single data collection from data directory.
723 path = fGridDataDir;
923e2ca5 724 if (!DirectoryExists(path)) {
d2a409b2 725 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
726 return kFALSE;
727 }
923e2ca5 728// CdWork();
d2a409b2 729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 731 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 732 command = "find ";
733 command += options;
734 command += path;
735 command += " ";
736 command += pattern;
737 command += conditions;
84fcd93f 738 printf("command: %s\n", command.Data());
d2a409b2 739 TGridResult *res = gGrid->Command(command);
740 if (res) delete res;
741 // Write standard output to file
742 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 743 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
744 Bool_t nullFile = kFALSE;
745 if (!hasGrep) {
d3339be3 746 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
747 } else {
a2f5fc01 748 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
749 if (nullFile) {
defd7a3a 750 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
751 return kFALSE;
752 }
d3339be3 753 }
0f389141 754 }
755 Bool_t fileExists = FileExists(file);
756 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 757 // Copy xml file to alien space
0f389141 758 if (fileExists) gGrid->Rm(file);
d2a409b2 759 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
760 if (!FileExists(file)) {
761 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
762 return kFALSE;
763 }
764 // Update list of files to be processed.
765 }
766 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
767 return kTRUE;
768 }
c57f56b7 769 // Several runs
a2f5fc01 770 Bool_t nullResult = kTRUE;
a8739e8a 771 if (fRunNumbers.Length()) {
772 TObjArray *arr = fRunNumbers.Tokenize(" ");
773 TObjString *os;
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 777 if (!DirectoryExists(path)) continue;
778// CdWork();
a8739e8a 779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
780 else file = Form("%s.xml", os->GetString().Data());
319593fb 781 // If local collection file does not exist, create it via 'find' command.
d3339be3 782 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 783 command = "find ";
784 command += options;
785 command += path;
786 command += pattern;
787 command += conditions;
788 TGridResult *res = gGrid->Command(command);
789 if (res) delete res;
790 // Write standard output to file
791 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 792 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
793 Bool_t nullFile = kFALSE;
794 if (!hasGrep) {
d3339be3 795 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
796 } else {
a2f5fc01 797 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
798 if (nullFile) {
defd7a3a 799 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
800 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
801 continue;
802 }
d3339be3 803 }
a2f5fc01 804 nullResult = kFALSE;
d3339be3 805 }
a8739e8a 806 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 807 // Check if there is one run per master job.
808 if (fNrunsPerMaster<2) {
809 if (FileExists(file)) {
0f389141 810 if (fOverwriteMode) gGrid->Rm(file);
811 else {
812 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
813 continue;
814 }
319593fb 815 }
816 // Copy xml file to alien space
817 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
818 if (!FileExists(file)) {
819 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
820 delete arr;
821 return kFALSE;
822 }
823 } else {
824 nruns++;
825 if (((nruns-1)%fNrunsPerMaster) == 0) {
826 schunk = os->GetString();
827 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
828 } else {
829 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 830 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 831 cbase->Add(cadd);
832 delete cadd;
833 }
834 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
835 continue;
836 }
837 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 838 if (FileExists(schunk)) {
839 if (fOverwriteMode) gGrid->Rm(file);
840 else {
841 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
842 continue;
843 }
319593fb 844 }
84fcd93f 845 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 846 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 847 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 848 if (!FileExists(schunk)) {
849 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
850 delete arr;
851 return kFALSE;
852 }
d3339be3 853 }
a8739e8a 854 }
855 delete arr;
a2f5fc01 856 if (nullResult) {
d3339be3 857 Error("CreateDataset", "No valid dataset corresponding to the query!");
858 return kFALSE;
859 }
a8739e8a 860 } else {
861 // Process a full run range.
862 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 863 format = Form("%%s/%s ", fRunPrefix.Data());
864 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 865 if (!DirectoryExists(path)) continue;
866// CdWork();
74013535 867 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 868 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 869 else file = Form(format.Data(), irun);
0f389141 870 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
871 if (fOverwriteMode) gGrid->Rm(file);
872 else {
873 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
874 continue;
875 }
a8739e8a 876 }
319593fb 877 // If local collection file does not exist, create it via 'find' command.
d3339be3 878 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 879 command = "find ";
880 command += options;
881 command += path;
882 command += pattern;
883 command += conditions;
884 TGridResult *res = gGrid->Command(command);
885 if (res) delete res;
886 // Write standard output to file
887 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 888 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
889 Bool_t nullFile = kFALSE;
890 if (!hasGrep) {
d3339be3 891 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
892 } else {
a2f5fc01 893 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
894 if (nullFile) {
defd7a3a 895 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
896 continue;
897 }
d3339be3 898 }
a2f5fc01 899 nullResult = kFALSE;
319593fb 900 }
a8739e8a 901 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 902 // Check if there is one run per master job.
903 if (fNrunsPerMaster<2) {
904 if (FileExists(file)) {
0f389141 905 if (fOverwriteMode) gGrid->Rm(file);
906 else {
907 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
908 continue;
909 }
319593fb 910 }
911 // Copy xml file to alien space
912 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
913 if (!FileExists(file)) {
914 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
915 return kFALSE;
916 }
917 } else {
918 nruns++;
95e5b448 919 // Check if the collection for the chunk exist locally.
920 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 921 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
922 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
923 else continue;
924 }
84fcd93f 925 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 926 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 927 schunk = Form(fRunPrefix.Data(), irun);
319593fb 928 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
929 } else {
930 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 cbase->Add(cadd);
932 delete cadd;
933 }
74013535 934 format = Form("%%s_%s.xml", fRunPrefix.Data());
935 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 936 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 937 continue;
938 }
904f9f5f 939 schunk = schunk2;
319593fb 940 if (FileExists(schunk)) {
0f389141 941 if (fOverwriteMode) gGrid->Rm(schunk);
942 else {
943 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
944 continue;
945 }
319593fb 946 }
84fcd93f 947 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 948 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 949 if (FileExists(schunk)) {
0f389141 950 if (fOverwriteMode) gGrid->Rm(schunk);
951 else {
952 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
953 continue;
954 }
95e5b448 955 }
319593fb 956 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
957 if (!FileExists(schunk)) {
958 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
959 return kFALSE;
960 }
961 }
c57f56b7 962 }
a2f5fc01 963 if (nullResult) {
d3339be3 964 Error("CreateDataset", "No valid dataset corresponding to the query!");
965 return kFALSE;
966 }
a8739e8a 967 }
c57f56b7 968 return kTRUE;
969}
970
971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CreateJDL()
973{
974// Generate a JDL file according to current settings. The name of the file is
975// specified by fJDLName.
976 Bool_t error = kFALSE;
977 TObjArray *arr = 0;
978 Bool_t copy = kTRUE;
d3b18c4c 979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 980 Bool_t generate = kTRUE;
981 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
982 if (!Connect()) {
983 Error("CreateJDL", "Alien connection required");
984 return kFALSE;
985 }
986 // Check validity of alien workspace
d3b18c4c 987 TString workdir;
988 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
989 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 990 workdir += fGridWorkingDir;
991 if (generate) {
992 TObjString *os;
993 if (!fInputFiles) {
994 Error("CreateJDL()", "Define some input files for your analysis.");
995 error = kTRUE;
996 }
997 // Compose list of input files
998 // Check if output files were defined
999 if (!fOutputFiles.Length()) {
1000 Error("CreateJDL", "You must define at least one output file");
1001 error = kTRUE;
1002 }
1003 // Check if an output directory was defined and valid
1004 if (!fGridOutputDir.Length()) {
1005 Error("CreateJDL", "You must define AliEn output directory");
1006 error = kTRUE;
1007 } else {
d3b18c4c 1008 if (!fProductionMode) {
1009 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1010 if (!DirectoryExists(fGridOutputDir)) {
1011 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1012 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1013 } else {
1014 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1015 // error = kTRUE;
1016 }
67a3b254 1017 } else {
1018 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1019 }
1020 gGrid->Cd(workdir);
1021 }
c57f56b7 1022 }
1023 // Exit if any error up to now
1024 if (error) return kFALSE;
1025 // Set JDL fields
0f389141 1026 if (!fUser.IsNull()) {
1027 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1028 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1029 }
1030 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1031 TString mergeExec = fExecutable;
1032 mergeExec.ReplaceAll(".sh", "_merge.sh");
1033 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1034 mergeExec.ReplaceAll(".sh", ".C");
1035 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1036 if (!fArguments.IsNull())
1037 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1038 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1039 else {
1040 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1041 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1042 }
b3e07543 1043
1f0d1ca2 1044 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1045 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1046 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048
0f389141 1049 if (fMaxInitFailed > 0) {
c57f56b7 1050 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1051 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1052 }
1053 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1054 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1055 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1056 }
1057 if (!IsOneStageMerging()) {
f47d5cb4 1058 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1059 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1060 }
1061 if (fSplitMode.Length()) {
c57f56b7 1062 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1063 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1064 }
1065 fMergingJDL->SetValue("Split", "\"se\"");
1066 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1067 if (!fAliROOTVersion.IsNull()) {
1068 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1069 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1070 }
1071 if (!fROOTVersion.IsNull()) {
c57f56b7 1072 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1073 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1074 }
1075 if (!fAPIVersion.IsNull()) {
c57f56b7 1076 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1077 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1078 }
648174cf 1079 if (!fExternalPackages.IsNull()) {
1080 arr = fExternalPackages.Tokenize(" ");
1081 TIter next(arr);
1082 while ((os=(TObjString*)next())) {
1083 TString pkgname = os->GetString();
1084 Int_t index = pkgname.Index("::");
1085 TString pkgversion = pkgname(index+2, pkgname.Length());
1086 pkgname.Remove(index);
1087 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1088 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1089 }
1090 delete arr;
1091 }
0f389141 1092 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1093 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1094 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1096 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1097 TString analysisFile = fExecutable;
1098 analysisFile.ReplaceAll(".sh", ".root");
1099 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1100 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1101 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1102 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1103 if (fAdditionalLibs.Length()) {
1104 arr = fAdditionalLibs.Tokenize(" ");
1105 TIter next(arr);
1106 while ((os=(TObjString*)next())) {
1107 if (os->GetString().Contains(".so")) continue;
1108 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1109 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1110 }
1111 delete arr;
1112 }
4e5c5506 1113 if (fPackages) {
1114 TIter next(fPackages);
1115 TObject *obj;
0f389141 1116 while ((obj=next())) {
4e5c5506 1117 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1118 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1119 }
4e5c5506 1120 }
c57f56b7 1121 if (fOutputArchive.Length()) {
1122 arr = fOutputArchive.Tokenize(" ");
1123 TIter next(arr);
0f389141 1124 Bool_t first = kTRUE;
1125 const char *comment = "Files to be archived";
1126 const char *comment1 = comment;
1127 while ((os=(TObjString*)next())) {
1128 if (!first) comment = NULL;
1129 if (!os->GetString().Contains("@") && fCloseSE.Length())
1130 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1131 else
1132 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1133 first = kFALSE;
1134 }
c57f56b7 1135 delete arr;
f790bc1b 1136 // Output archive for the merging jdl
1137 TString outputArchive;
1138 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1139 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1140 // Add normal output files, extra files + terminate files
1141 TString files = GetListOfFiles("outextter");
1142 // Do not register merge excludes
1143 if (!fMergeExcludes.IsNull()) {
1144 arr = fMergeExcludes.Tokenize(" ");
1145 TIter next1(arr);
1146 while ((os=(TObjString*)next1())) {
1147 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1148 files.ReplaceAll(os->GetString(),"");
1149 }
66d71516 1150 delete arr;
0f389141 1151 }
f790bc1b 1152 files.ReplaceAll(".root", "*.root");
b83f84f0 1153 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1154 } else {
95f84779 1155 TString files = fOutputArchive;
1156 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1157 outputArchive = files;
f790bc1b 1158 }
0f389141 1159 arr = outputArchive.Tokenize(" ");
705adb3e 1160 TIter next2(arr);
0f389141 1161 comment = comment1;
1162 first = kTRUE;
705adb3e 1163 while ((os=(TObjString*)next2())) {
0f389141 1164 if (!first) comment = NULL;
0cdf65a8 1165 TString currentfile = os->GetString();
0cdf65a8 1166 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1167 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1168 else
0cdf65a8 1169 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1170 first = kFALSE;
1171 }
1172 delete arr;
c57f56b7 1173 }
149d288c 1174 arr = fOutputFiles.Tokenize(",");
c57f56b7 1175 TIter next(arr);
0f389141 1176 Bool_t first = kTRUE;
f790bc1b 1177 const char *comment = "Files to be saved";
43da816a 1178 while ((os=(TObjString*)next())) {
e1eaf596 1179 // Ignore ouputs in jdl that are also in outputarchive
1180 TString sout = os->GetString();
f790bc1b 1181 sout.ReplaceAll("*", "");
1182 sout.ReplaceAll(".root", "");
e1eaf596 1183 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1184 if (fOutputArchive.Contains(sout)) continue;
0f389141 1185 if (!first) comment = NULL;
43da816a 1186 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1187 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1188 else
0f389141 1189 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1190 first = kFALSE;
f790bc1b 1191 if (fMergeExcludes.Contains(sout)) continue;
1192 if (!os->GetString().Contains("@") && fCloseSE.Length())
1193 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1194 else
1195 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1196 }
c57f56b7 1197 delete arr;
0f389141 1198 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1199 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1200 TString validationScript = fValidationScript;
0f389141 1201 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1202 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1203 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1204 if (fMasterResubmitThreshold) {
1205 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1206 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1207 }
d2a409b2 1208 // Write a jdl with 2 input parameters: collection name and output dir name.
1209 WriteJDL(copy);
c57f56b7 1210 }
1211 // Copy jdl to grid workspace
a8739e8a 1212 if (copy) {
b5e4aaa7 1213 // Check if an output directory was defined and valid
1214 if (!fGridOutputDir.Length()) {
1215 Error("CreateJDL", "You must define AliEn output directory");
1216 return kFALSE;
1217 } else {
1218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1219 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1220 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1221 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1222 } else {
1223 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 }
1227 gGrid->Cd(workdir);
1228 }
648174cf 1229 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1230 TString mergeJDLName = fExecutable;
1231 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1232 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1233 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1234 if (fProductionMode) {
648174cf 1235 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1236 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1237 }
648174cf 1238 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1239 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1240 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1241 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1242 if (fMergeViaJDL) {
1243 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1244 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1245 }
648174cf 1246 }
c57f56b7 1247 if (fAdditionalLibs.Length()) {
e7c71df0 1248 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
c57f56b7 1252 if (os->GetString().Contains(".so")) continue;
4e5c5506 1253 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1254 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1255 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1256 }
1257 delete arr;
1258 }
4e5c5506 1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 while ((obj=next())) {
fdbbc7be 1263 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1265 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1266 }
1267 }
c57f56b7 1268 }
1269 return kTRUE;
1270}
1271
a8739e8a 1272//______________________________________________________________________________
d2a409b2 1273Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1274{
1275// Writes one or more JDL's corresponding to findex. If findex is negative,
1276// all run numbers are considered in one go (jdl). For non-negative indices
1277// they correspond to the indices in the array fInputFiles.
1278 if (!fInputFiles) return kFALSE;
d3b18c4c 1279 TObject *os;
1280 TString workdir;
1281 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1282 workdir += fGridWorkingDir;
f47d5cb4 1283 TString stageName = "$2";
1284 if (fProductionMode) stageName = "$4";
1285 if (!fMergeDirName.IsNull()) {
1286 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1287 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1288 } else {
1289 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1290 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1291 }
d3b18c4c 1292 if (fProductionMode) {
a8739e8a 1293 TIter next(fInputFiles);
b3e07543 1294 while ((os=next())) {
d3b18c4c 1295 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1296 }
d3b18c4c 1297 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1298 } else {
1299 if (!fRunNumbers.Length() && !fRunRange[0]) {
1300 // One jdl with no parameters in case input data is specified by name.
1301 TIter next(fInputFiles);
1302 while ((os=next()))
1303 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1304 if (!fOutputSingle.IsNull())
1305 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1306 else {
1307 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1308 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1309 }
1310 } else {
1311 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1312 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1313 if (!fOutputSingle.IsNull()) {
1314 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1315 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1316 } else {
1317 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1318 }
1319 }
a8739e8a 1320 }
1321
a8739e8a 1322 // Generate the JDL as a string
1323 TString sjdl = fGridJDL->Generate();
0f389141 1324 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1325 // Final merge jdl
1326 if (!fMergeDirName.IsNull()) {
1327 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1328 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1329 } else {
1330 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1331 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1332 }
b3e07543 1333 TString sjdl2 = fMergingJDL->Generate();
1334 Int_t index, index1;
a8739e8a 1335 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1336 sjdl.ReplaceAll("(member", "\n (member");
1337 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1338 sjdl.ReplaceAll("{", "{\n ");
1339 sjdl.ReplaceAll("};", "\n};");
1340 sjdl.ReplaceAll("{\n \n", "{\n");
1341 sjdl.ReplaceAll("\n\n", "\n");
1342 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1343 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1344 sjdl1.ReplaceAll("(member", "\n (member");
1345 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1346 sjdl1.ReplaceAll("{", "{\n ");
1347 sjdl1.ReplaceAll("};", "\n};");
1348 sjdl1.ReplaceAll("{\n \n", "{\n");
1349 sjdl1.ReplaceAll("\n\n", "\n");
1350 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1351 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1352 sjdl2.ReplaceAll("(member", "\n (member");
1353 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1354 sjdl2.ReplaceAll("{", "{\n ");
1355 sjdl2.ReplaceAll("};", "\n};");
1356 sjdl2.ReplaceAll("{\n \n", "{\n");
1357 sjdl2.ReplaceAll("\n\n", "\n");
1358 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1359 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1360 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1361 index = sjdl.Index("JDLVariables");
1362 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1363 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1364 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1365 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1366 index = fJobTag.Index(":");
1367 if (index < 0) index = fJobTag.Length();
1368 TString jobTag = fJobTag;
f47d5cb4 1369 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1370 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1371 if (fProductionMode) {
1372 sjdl1.Prepend("# Generated merging jdl (production mode) \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = train number \
1375 \n# $3 = production (like LHC10b) \
1376 \n# $4 = merging stage \
1377 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1378 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1379 sjdl2.Prepend("# Generated merging jdl \
1380 \n# $1 = full alien path to output directory to be merged \
1381 \n# $2 = train number \
1382 \n# $3 = production (like LHC10b) \
1383 \n# $4 = merging stage \
1384 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1385 } else {
1386 sjdl1.Prepend("# Generated merging jdl \
1387 \n# $1 = full alien path to output directory to be merged \
1388 \n# $2 = merging stage \
03335324 1389 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1390 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1391 sjdl2.Prepend("# Generated merging jdl \
1392 \n# $1 = full alien path to output directory to be merged \
1393 \n# $2 = merging stage \
03335324 1394 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1395 }
0f389141 1396 index = sjdl1.Index("JDLVariables");
1397 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1398 index = sjdl2.Index("JDLVariables");
1399 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1400 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1401 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1402 index = sjdl2.Index("Split =");
1403 if (index>=0) {
1404 index1 = sjdl2.Index("\n", index);
1405 sjdl2.Remove(index, index1-index+1);
1406 }
1407 index = sjdl2.Index("SplitMaxInputFileNumber");
1408 if (index>=0) {
1409 index1 = sjdl2.Index("\n", index);
1410 sjdl2.Remove(index, index1-index+1);
1411 }
1412 index = sjdl2.Index("InputDataCollection");
1413 if (index>=0) {
1414 index1 = sjdl2.Index(";", index);
1415 sjdl2.Remove(index, index1-index+1);
1416 }
1417 index = sjdl2.Index("InputDataListFormat");
1418 if (index>=0) {
1419 index1 = sjdl2.Index("\n", index);
1420 sjdl2.Remove(index, index1-index+1);
1421 }
1422 index = sjdl2.Index("InputDataList");
1423 if (index>=0) {
1424 index1 = sjdl2.Index("\n", index);
1425 sjdl2.Remove(index, index1-index+1);
1426 }
f47d5cb4 1427 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1428 // Write jdl to file
a8739e8a 1429 ofstream out;
d2a409b2 1430 out.open(fJDLName.Data(), ios::out);
a8739e8a 1431 if (out.bad()) {
d3b18c4c 1432 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1433 return kFALSE;
1434 }
1435 out << sjdl << endl;
b3e07543 1436 out.close();
0f389141 1437 TString mergeJDLName = fExecutable;
1438 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1439 if (fMergeViaJDL) {
1440 ofstream out1;
1441 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1442 if (out1.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1444 return kFALSE;
1445 }
1446 out1 << sjdl1 << endl;
b3e07543 1447 out1.close();
1448 ofstream out2;
1449 TString finalJDL = mergeJDLName;
1450 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1451 out2.open(finalJDL.Data(), ios::out);
1452 if (out2.bad()) {
1453 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1454 return kFALSE;
1455 }
1456 out2 << sjdl2 << endl;
1457 out2.close();
0f389141 1458 }
a8739e8a 1459
1460 // Copy jdl to grid workspace
1461 if (!copy) {
d3b18c4c 1462 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1463 } else {
d2a409b2 1464 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1465 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1466 TString finalJDL = mergeJDLName;
1467 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1468 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1469 if (fProductionMode) {
b5fe9cba 1470 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1471 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1472 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1473 }
d2a409b2 1474 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1475 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1476 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1477 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1478 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1479 if (fMergeViaJDL) {
b3e07543 1480 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1481 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1482 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1483 }
a8739e8a 1484 }
1485 return kTRUE;
1486}
1487
c57f56b7 1488//______________________________________________________________________________
5513444a 1489Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1490{
1491// Returns true if file exists.
5513444a 1492 if (!gGrid) return kFALSE;
2ea55496 1493 TString slfn = lfn;
1494 slfn.ReplaceAll("alien://","");
1495 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1496 if (!res) return kFALSE;
1497 TMap *map = dynamic_cast<TMap*>(res->At(0));
1498 if (!map) {
1499 delete res;
1500 return kFALSE;
1501 }
1502 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1503 if (!objs || !objs->GetString().Length()) {
1504 delete res;
1505 return kFALSE;
1506 }
1507 delete res;
1508 return kTRUE;
1509}
1510
923e2ca5 1511//______________________________________________________________________________
1512Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1513{
1514// Returns true if directory exists. Can be also a path.
1515 if (!gGrid) return kFALSE;
1516 // Check if dirname is a path
1517 TString dirstripped = dirname;
1518 dirstripped = dirstripped.Strip();
1519 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1520 TString dir = gSystem->BaseName(dirstripped);
1521 dir += "/";
1522 TString path = gSystem->DirName(dirstripped);
1523 TGridResult *res = gGrid->Ls(path, "-F");
1524 if (!res) return kFALSE;
1525 TIter next(res);
1526 TMap *map;
1527 TObject *obj;
1528 while ((map=dynamic_cast<TMap*>(next()))) {
1529 obj = map->GetValue("name");
1530 if (!obj) break;
1531 if (dir == obj->GetName()) {
1532 delete res;
1533 return kTRUE;
1534 }
1535 }
1536 delete res;
1537 return kFALSE;
1538}
1539
c57f56b7 1540//______________________________________________________________________________
a2f5fc01 1541void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1542{
1543// Check input data type.
a2f5fc01 1544 isCollection = kFALSE;
1545 isXml = kFALSE;
1546 useTags = kFALSE;
c57f56b7 1547 if (!gGrid) {
1548 Error("CheckDataType", "No connection to grid");
1549 return;
1550 }
a2f5fc01 1551 isCollection = IsCollection(lfn);
c57f56b7 1552 TString msg = "\n##### file: ";
1553 msg += lfn;
a2f5fc01 1554 if (isCollection) {
c57f56b7 1555 msg += " type: raw_collection;";
1556 // special treatment for collections
a2f5fc01 1557 isXml = kFALSE;
c57f56b7 1558 // check for tag files in the collection
1559 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1560 if (!res) {
1561 msg += " using_tags: No (unknown)";
23329835 1562 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1563 return;
1564 }
1565 const char* typeStr = res->GetKey(0, "origLFN");
1566 if (!typeStr || !strlen(typeStr)) {
1567 msg += " using_tags: No (unknown)";
23329835 1568 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1569 return;
1570 }
1571 TString file = typeStr;
a2f5fc01 1572 useTags = file.Contains(".tag");
1573 if (useTags) msg += " using_tags: Yes";
c57f56b7 1574 else msg += " using_tags: No";
23329835 1575 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1576 return;
1577 }
1578 TString slfn(lfn);
1579 slfn.ToLower();
a2f5fc01 1580 isXml = slfn.Contains(".xml");
1581 if (isXml) {
c57f56b7 1582 // Open xml collection and check if there are tag files inside
1583 msg += " type: xml_collection;";
1584 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1585 if (!coll) {
1586 msg += " using_tags: No (unknown)";
23329835 1587 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1588 return;
1589 }
1590 TMap *map = coll->Next();
1591 if (!map) {
1592 msg += " using_tags: No (unknown)";
23329835 1593 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1594 return;
1595 }
1596 map = (TMap*)map->GetValue("");
1597 TString file;
1598 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1599 useTags = file.Contains(".tag");
c57f56b7 1600 delete coll;
a2f5fc01 1601 if (useTags) msg += " using_tags: Yes";
c57f56b7 1602 else msg += " using_tags: No";
23329835 1603 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1604 return;
1605 }
a2f5fc01 1606 useTags = slfn.Contains(".tag");
c57f56b7 1607 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1608 else msg += " type: unknown file;";
a2f5fc01 1609 if (useTags) msg += " using_tags: Yes";
c57f56b7 1610 else msg += " using_tags: No";
23329835 1611 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1612}
1613
4e5c5506 1614//______________________________________________________________________________
1615void AliAnalysisAlien::EnablePackage(const char *package)
1616{
1617// Enables a par file supposed to exist in the current directory.
1618 TString pkg(package);
1619 pkg.ReplaceAll(".par", "");
1620 pkg += ".par";
1621 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1622 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1623 return;
1624 }
1625 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1626 Info("EnablePackage", "AliEn plugin will use .par packages");
1627 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1628 if (!fPackages) {
1629 fPackages = new TObjArray();
1630 fPackages->SetOwner();
1631 }
1632 fPackages->Add(new TObjString(pkg));
1633}
1634
3bdcb562 1635//______________________________________________________________________________
1636TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1637{
1638// Make a tree from files having the location specified in fFileForTestMode.
1639// Inspired from JF's CreateESDChain.
1640 if (fFileForTestMode.IsNull()) {
1641 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1642 return NULL;
1643 }
1644 if (gSystem->AccessPathName(fFileForTestMode)) {
1645 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1646 return NULL;
1647 }
1648 // Open the file
1649 ifstream in;
1650 in.open(fFileForTestMode);
1651 Int_t count = 0;
1652 // Read the input list of files and add them to the chain
1653 TString line;
1654 TChain *chain = new TChain(treeName);
1655 while (in.good())
1656 {
1657 in >> line;
1658 if (line.IsNull()) continue;
1659 if (count++ == fNtestFiles) break;
1660 TString esdFile(line);
1661 TFile *file = TFile::Open(esdFile);
1662 if (file) {
1663 if (!file->IsZombie()) chain->Add(esdFile);
1664 file->Close();
1665 } else {
1666 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1667 }
1668 }
1669 in.close();
1670 if (!chain->GetListOfFiles()->GetEntries()) {
1671 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1672 delete chain;
1673 return NULL;
1674 }
1675// chain->ls();
1676 return chain;
1677}
1678
d2a409b2 1679//______________________________________________________________________________
1680const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1681{
1682// Get job status for all jobs with jobid>jobidstart.
1683 static char mstatus[20];
1684 mstatus[0] = '\0';
1685 nrunning = 0;
1686 nwaiting = 0;
1687 nerror = 0;
1688 ndone = 0;
1689 TGridJobStatusList *list = gGrid->Ps("");
1690 if (!list) return mstatus;
1691 Int_t nentries = list->GetSize();
1692 TGridJobStatus *status;
1693 Int_t pid;
1694 for (Int_t ijob=0; ijob<nentries; ijob++) {
1695 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1696 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1697 if (pid<jobidstart) continue;
1698 if (pid == lastid) {
8ddad121 1699 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1700 }
1701 switch (status->GetStatus()) {
1702 case TGridJobStatus::kWAITING:
1703 nwaiting++; break;
1704 case TGridJobStatus::kRUNNING:
1705 nrunning++; break;
1706 case TGridJobStatus::kABORTED:
1707 case TGridJobStatus::kFAIL:
1708 case TGridJobStatus::kUNKNOWN:
1709 nerror++; break;
1710 case TGridJobStatus::kDONE:
1711 ndone++;
1712 }
1713 }
1714 list->Delete();
1715 delete list;
1716 return mstatus;
1717}
1718
c57f56b7 1719//______________________________________________________________________________
1720Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1721{
1722// Returns true if file is a collection. Functionality duplicated from
1723// TAlien::Type() because we don't want to directly depend on TAlien.
1724 if (!gGrid) {
1725 Error("IsCollection", "No connection to grid");
1726 return kFALSE;
1727 }
1728 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1729 if (!res) return kFALSE;
1730 const char* typeStr = res->GetKey(0, "type");
1731 if (!typeStr || !strlen(typeStr)) return kFALSE;
1732 if (!strcmp(typeStr, "collection")) return kTRUE;
1733 delete res;
1734 return kFALSE;
1735}
1736
fe2d7fc2 1737//______________________________________________________________________________
1738Bool_t AliAnalysisAlien::IsSingleOutput() const
1739{
1740// Check if single-ouput option is on.
1741 return (!fOutputSingle.IsNull());
1742}
1743
16a4353c 1744//______________________________________________________________________________
1745void AliAnalysisAlien::Print(Option_t *) const
1746{
1747// Print current plugin settings.
84fcd93f 1748 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1750 if (mgr && mgr->IsProofMode()) {
1751 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1752 if (TestBit(AliAnalysisGrid::kTest))
1753 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1754 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1755 if (!fProofDataSet.IsNull())
1756 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1757 if (fProofReset==1)
1758 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1759 if (fProofReset>1)
1760 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (!fRootVersionForProof.IsNull())
1762 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1763 else
1764 printf("= ROOT version requested________________________ default\n");
1765 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1766 if (!fAliRootMode.IsNull())
1767 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1768 if (fNproofWorkers)
1769 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1770 if (fNproofWorkersPerSlave)
1771 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1772 if (TestSpecialBit(kClearPackages))
1773 printf("= ClearPackages requested...\n");
1774 if (fIncludePath.Data())
1775 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1776 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1777 if (fPackages && fPackages->GetEntries()) {
1778 TIter next(fPackages);
1779 TObject *obj;
1780 TString list;
1781 while ((obj=next())) list += obj->GetName();
1782 printf("= Par files to be used: ________________________ %s\n", list.Data());
1783 }
1784 if (TestSpecialBit(kProofConnectGrid))
1785 printf("= Requested PROOF connection to grid\n");
1786 return;
1787 }
e1c22e21 1788 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1789 if (fOverwriteMode) {
1790 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1791 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1792 }
348be253 1793 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1794 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1795 printf("= Production mode:______________________________ %d\n", fProductionMode);
1796 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1797 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1798 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1799 if (fUser.Length())
84fcd93f 1800 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1801 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1802 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1803 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1804 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1805 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1806 if (fRunNumbers.Length())
84fcd93f 1807 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1808 if (fRunRange[0])
74013535 1809 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1810 if (!fRunRange[0] && !fRunNumbers.Length()) {
1811 TIter next(fInputFiles);
1812 TObject *obj;
1813 TString list;
1814 while ((obj=next())) list += obj->GetName();
84fcd93f 1815 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1816 }
1817 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1818 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1819 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1820 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1821 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1822 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1823 printf("=====================================================================\n");
1824 printf("= Job price: ___________________________________ %d\n", fPrice);
1825 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1826 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1827 if (fMaxInitFailed>0)
84fcd93f 1828 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1829 if (fMasterResubmitThreshold>0)
84fcd93f 1830 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1831 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1832 if (fNrunsPerMaster>0)
84fcd93f 1833 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1834 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1835 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1836 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1837 if (fArguments.Length())
84fcd93f 1838 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1839 if (fExecutableArgs.Length())
1840 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1841 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1842 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1843 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1844 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1845 if (fDatasetName)
84fcd93f 1846 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1847 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1848 if (fIncludePath.Data())
84fcd93f 1849 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1850 if (fCloseSE.Length())
84fcd93f 1851 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1852 if (fFriendChainName.Length())
84fcd93f 1853 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1854 if (fPackages && fPackages->GetEntries()) {
16a4353c 1855 TIter next(fPackages);
1856 TObject *obj;
1857 TString list;
1858 while ((obj=next())) list += obj->GetName();
84fcd93f 1859 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1860 }
1861}
1862
c57f56b7 1863//______________________________________________________________________________
1864void AliAnalysisAlien::SetDefaults()
1865{
1866// Set default values for everything. What cannot be filled will be left empty.
1867 if (fGridJDL) delete fGridJDL;
1868 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1869 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1870 fPrice = 1;
1871 fTTL = 30000;
1872 fSplitMaxInputFileNumber = 100;
1873 fMaxInitFailed = 0;
1874 fMasterResubmitThreshold = 0;
bb885a9e 1875 fNtestFiles = 10;
149d288c 1876 fNreplicas = 2;
a8739e8a 1877 fRunRange[0] = 0;
1878 fRunRange[1] = 0;
8e3bd06a 1879 fRunPrefix = "%d";
319593fb 1880 fNrunsPerMaster = 1;
16a4353c 1881 fMaxMergeFiles = 100;
c57f56b7 1882 fRunNumbers = "";
1883 fExecutable = "analysis.sh";
0a1c1f7f 1884 fExecutableCommand = "root -b -q";
c57f56b7 1885 fArguments = "";
631c0b05 1886 fExecutableArgs = "";
c57f56b7 1887 fAnalysisMacro = "myAnalysis.C";
1888 fAnalysisSource = "";
1889 fAdditionalLibs = "";
1890 fSplitMode = "se";
1891 fAPIVersion = "";
1892 fROOTVersion = "";
1893 fAliROOTVersion = "";
1894 fUser = ""; // Your alien user name
1895 fGridWorkingDir = "";
1896 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1897 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1898 fFriendChainName = "";
c57f56b7 1899 fGridOutputDir = "output";
b83f84f0 1900 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1901 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1902 fInputFormat = "xml-single";
1903 fJDLName = "analysis.jdl";
c6cb3634 1904 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1905 fMergeExcludes = "";
0f389141 1906 fMergeViaJDL = 0;
348be253 1907 SetUseCopy(kTRUE);
1908 SetCheckCopy(kTRUE);
149d288c 1909 SetDefaultOutputs(kTRUE);
e1c22e21 1910 fOverwriteMode = 1;
c57f56b7 1911}
1912
0f389141 1913//______________________________________________________________________________
b3e07543 1914Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1915{
b3e07543 1916// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1917 // First check if the result is already in the output directory.
1918 if (FileExists(Form("%s/%s",aliendir,filename))) {
1919 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1920 return kFALSE;
1921 }
b3e07543 1922 // Now check the last stage done.
1923 Int_t stage = 0;
1924 while (1) {
1925 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1926 stage++;
1927 }
1928 // Next stage of merging
1929 stage++;
1930 TString pattern = "*root_archive.zip";
1931 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1932 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1933 if (res) delete res;
1934 // Write standard output to file
1935 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1936 // Count the number of files inside
1937 ifstream ifile;
1938 ifile.open(Form("Stage_%d.xml",stage));
1939 if (!ifile.good()) {
1940 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1941 return kFALSE;
b3e07543 1942 }
1943 TString line;
1944 Int_t nfiles = 0;
1945 while (!ifile.eof()) {
1946 ifile >> line;
1947 if (line.Contains("/event")) nfiles++;
7c2cd90a 1948 }
b3e07543 1949 ifile.close();
1950 if (!nfiles) {
1951 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1952 return kFALSE;
1953 } else {
1954 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1955 }
b3e07543 1956 // Copy the file in the output directory
1957 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1958 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1959 // Check if this is the last stage to be done.
1960 Bool_t laststage = (nfiles<nperchunk);
1961 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1962 if (laststage) {
1963 printf("### Submiting final merging stage %d\n", stage);
1964 TString finalJDL = jdl;
1965 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1966 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1967 Int_t jobId = SubmitSingleJob(query);
1968 if (!jobId) return kFALSE;
1969 } else {
1970 printf("### Submiting merging stage %d\n", stage);
03335324 1971 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1972 Int_t jobId = SubmitSingleJob(query);
1973 if (!jobId) return kFALSE;
7c2cd90a 1974 }
b3e07543 1975 return kTRUE;
1976}
7c2cd90a 1977
1978//______________________________________________________________________________
1979Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1980{
1981// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1982 if (!gGrid) return 0;
1983 printf("=> %s ------> ",query);
1984 TGridResult *res = gGrid->Command(query);
1985 if (!res) return 0;
1986 TString jobId = res->GetKey(0,"jobId");
1987 delete res;
1988 if (jobId.IsNull()) {
1989 printf("submission failed. Reason:\n");
1990 gGrid->Stdout();
1991 gGrid->Stderr();
1992 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1993 return 0;
1994 }
1995 printf(" Job id: %s\n", jobId.Data());
1996 return atoi(jobId);
1997}
1998
1999//______________________________________________________________________________
b3e07543 2000Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2001{
b3e07543 2002// Merge given output files from basedir. Basedir can be an alien output directory
2003// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2004// files in a group (ignored for xml input). Merging can be done in stages:
2005// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2006// stage=1 : works with an xml of all root_archive.zip in the output directory
2007// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2008 TString outputFile = output;
0f389141 2009 TString command;
a2f5fc01 2010 TString outputChunk;
2011 TString previousChunk = "";
b3e07543 2012 TObjArray *listoffiles = new TObjArray();
2013// listoffiles->SetOwner();
a2f5fc01 2014 Int_t countChunk = 0;
2015 Int_t countZero = nmaxmerge;
0f389141 2016 Bool_t merged = kTRUE;
a2f5fc01 2017 Int_t index = outputFile.Index("@");
2018 if (index > 0) outputFile.Remove(index);
7c2cd90a 2019 TString inputFile = outputFile;
b3e07543 2020 TString sbasedir = basedir;
2021 if (sbasedir.Contains(".xml")) {
2022 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2023 nmaxmerge = 9999999;
2024 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2025 if (!coll) {
2026 ::Error("MergeOutput", "Input XML collection empty.");
2027 return kFALSE;
2028 }
2029 // Iterate grid collection
2030 while (coll->Next()) {
2031 TString fname = gSystem->DirName(coll->GetTURL());
2032 fname += "/";
2033 fname += inputFile;
2034 listoffiles->Add(new TNamed(fname.Data(),""));
2035 }
2036 } else {
2037 command = Form("find %s/ *%s", basedir, inputFile.Data());
2038 printf("command: %s\n", command.Data());
2039 TGridResult *res = gGrid->Command(command);
2040 if (!res) {
2041 ::Error("MergeOutput","No result for the find command\n");
2042 delete listoffiles;
2043 return kFALSE;
2044 }
2045 TIter nextmap(res);
2046 TMap *map = 0;
2047 while ((map=(TMap*)nextmap())) {
2048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2049 if (!objs || !objs->GetString().Length()) {
2050 // Nothing found - skip this output
2051 delete res;
2052 delete listoffiles;
2053 return kFALSE;
2054 }
2055 listoffiles->Add(new TNamed(objs->GetName(),""));
2056 }
2057 delete res;
2058 }
2059 if (!listoffiles->GetEntries()) {
7c2cd90a 2060 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2061 delete listoffiles;
0f389141 2062 return kFALSE;
2063 }
2064
2065 TFileMerger *fm = 0;
b3e07543 2066 TIter next0(listoffiles);
2067 TObjArray *listoffilestmp = new TObjArray();
2068 listoffilestmp->SetOwner();
2069 TObject *nextfile;
2070 TString snextfile;
2071 // Keep only the files at upper level
2072 Int_t countChar = 0;
2073 while ((nextfile=next0())) {
2074 snextfile = nextfile->GetName();
2075 Int_t crtCount = snextfile.CountChar('/');
2076 if (nextfile == listoffiles->First()) countChar = crtCount;
2077 if (crtCount < countChar) countChar = crtCount;
2078 }
2079 next0.Reset();
2080 while ((nextfile=next0())) {
2081 snextfile = nextfile->GetName();
2082 Int_t crtCount = snextfile.CountChar('/');
2083 if (crtCount > countChar) {
2084 delete nextfile;
2085 continue;
2086 }
2087 listoffilestmp->Add(nextfile);
2088 }
2089 delete listoffiles;
2090 listoffiles = listoffilestmp; // Now contains 'good' files
2091 listoffiles->Print();
2092 TIter next(listoffiles);
7c2cd90a 2093 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2094 outputChunk = outputFile;
2095 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2096 // Check for existent temporary merge files
e1c22e21 2097 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2098 // Preserve old merging functionality for stage 0.
2099 if (stage==0) {
2100 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2101 while (1) {
2102 // Skip as many input files as in a chunk
2ea55496 2103 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2104 nextfile = next();
2105 if (!nextfile) {
2ea55496 2106 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2107 delete listoffiles;
2ea55496 2108 return kFALSE;
2109 }
b3e07543 2110 snextfile = nextfile->GetName();
7c2cd90a 2111 }
2112 outputChunk = outputFile;
2113 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2114 countChunk++;
2115 if (gSystem->AccessPathName(outputChunk)) continue;
2116 // Merged file with chunks up to <countChunk> found
2117 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2118 previousChunk = outputChunk;
2119 break;
2120 }
2121 }
2122 countZero = nmaxmerge;
2123
b3e07543 2124 while ((nextfile=next())) {
2125 snextfile = nextfile->GetName();
2ea55496 2126 // Loop 'find' results and get next LFN
7c2cd90a 2127 if (countZero == nmaxmerge) {
2128 // First file in chunk - create file merger and add previous chunk if any.
2129 fm = new TFileMerger(kFALSE);
2130 fm->SetFastMethod(kTRUE);
2131 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2132 outputChunk = outputFile;
2133 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2134 }
2135 // If last file found, put merged results in the output file
b3e07543 2136 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2137 // Add file to be merged and decrement chunk counter.
b3e07543 2138 fm->AddFile(snextfile);
7c2cd90a 2139 countZero--;
b3e07543 2140 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2141 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2142 // Nothing found - skip this output
2143 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2144 merged = kFALSE;
2145 break;
7c2cd90a 2146 }
2147 fm->OutputFile(outputChunk);
2148 // Merge the outputs, then go to next chunk
2149 if (!fm->Merge()) {
2150 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2151 merged = kFALSE;
2152 break;
7c2cd90a 2153 } else {
2154 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2155 gSystem->Unlink(previousChunk);
2156 }
b3e07543 2157 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2158 countChunk++;
2159 countZero = nmaxmerge;
2160 previousChunk = outputChunk;
0f389141 2161 }
0f389141 2162 }
b3e07543 2163 delete listoffiles;
2164 delete fm;
7c2cd90a 2165 return merged;
2166 }
2167 // Merging stage different than 0.
2168 // Move to the begining of the requested chunk.
7c2cd90a 2169 fm = new TFileMerger(kFALSE);
2170 fm->SetFastMethod(kTRUE);
b3e07543 2171 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2172 delete listoffiles;
7c2cd90a 2173 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2174 // Nothing found - skip this output
2175 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2176 delete fm;
2177 return kFALSE;
2178 }
b3e07543 2179 fm->OutputFile(outputFile);
7c2cd90a 2180 // Merge the outputs
2181 if (!fm->Merge()) {
2182 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2183 delete fm;
2184 return kFALSE;
2185 } else {
b3e07543 2186 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2187 }
7c2cd90a 2188 delete fm;
2189 return kTRUE;
0f389141 2190}
2191
c57f56b7 2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::MergeOutputs()
2194{
2195// Merge analysis outputs existing in the AliEn space.
2196 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2197 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2198 if (!Connect()) {
2199 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2200 return kFALSE;
0f389141 2201 }
c9e8f7fd 2202 if (fMergeViaJDL) {
2203 if (!TestBit(AliAnalysisGrid::kMerge)) {
2204 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2205 return kFALSE;
2206 }
2207 if (fProductionMode) {
2208 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2209 return kFALSE;
2210 }
0f389141 2211 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2212 if (!SubmitMerging()) return kFALSE;
0f389141 2213 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2214 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2215 return kFALSE;
c57f56b7 2216 }
2217 // Get the output path
d2a409b2 2218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2219 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2220 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2221 return kFALSE;
2222 }
2223 if (!fOutputFiles.Length()) {
2224 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2225 return kFALSE;
0f389141 2226 }
2227 // Check if fast read option was requested
e1c22e21 2228 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2229 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2230 if (fFastReadOption) {
d3339be3 2231 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2232 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2233 gEnv->SetValue("XNet.ConnectTimeout",50);
2234 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2235 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2236 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2237 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2238 }
e8b839ab 2239 // Make sure we change the temporary directory
2240 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2241 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2242 TIter next(list);
2243 TObjString *str;
a2f5fc01 2244 TString outputFile;
c57f56b7 2245 Bool_t merged = kTRUE;
2246 while((str=(TObjString*)next())) {
a2f5fc01 2247 outputFile = str->GetString();
2248 Int_t index = outputFile.Index("@");
2249 if (index > 0) outputFile.Remove(index);
2250 TString outputChunk = outputFile;
2251 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2252 // Skip already merged outputs
a2f5fc01 2253 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2254 if (fOverwriteMode) {
a2f5fc01 2255 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2256 gSystem->Unlink(outputFile);
2257 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2258 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2259 outputChunk.Data());
2260 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2261 }
2262 } else {
a2f5fc01 2263 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2264 continue;
2265 }
2266 } else {
a2f5fc01 2267 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2268 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2269 outputChunk.Data());
2270 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2271 }
2272 }
bb885a9e 2273 if (fMergeExcludes.Length() &&
a2f5fc01 2274 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2275 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2276 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2277 if (!merged) {
2278 Error("MergeOutputs", "Terminate() will NOT be executed");
2279 return kFALSE;
ff07ec61 2280 }
2281 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2282 if (fileOpened) fileOpened->Close();
c57f56b7 2283 }
0f389141 2284 return kTRUE;
c57f56b7 2285}
2286
bb885a9e 2287//______________________________________________________________________________
2288void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2289{
2290// Use the output files connected to output containers from the analysis manager
2291// rather than the files defined by SetOutputFiles
2292 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2293 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2294 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2295}
2296
149d288c 2297//______________________________________________________________________________
2298void AliAnalysisAlien::SetOutputFiles(const char *list)
2299{
2300// Manually set the output files list.
2301// Removes duplicates. Not allowed if default outputs are not disabled.
2302 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2303 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2304 return;
2305 }
2306 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2307 fOutputFiles = "";
2308 TString slist = list;
2309 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2310 TObjArray *arr = slist.Tokenize(" ");
2311 TObjString *os;
2312 TIter next(arr);
2313 TString sout;
2314 while ((os=(TObjString*)next())) {
2315 sout = os->GetString();
2316 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2317 if (fOutputFiles.Contains(sout)) continue;
2318 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2319 fOutputFiles += sout;
2320 }
2321 delete arr;
f790bc1b 2322}
149d288c 2323
2324//______________________________________________________________________________
2325void AliAnalysisAlien::SetOutputArchive(const char *list)
2326{
2327// Manually set the output archive list. Free text - you are on your own...
2328// Not allowed if default outputs are not disabled.
2329 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2330 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2331 return;
2332 }
2333 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2334 fOutputArchive = list;
2335}
2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2339{
2340// Setting a prefered output SE is not allowed anymore.
2341 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2342}
2343
5513444a 2344//______________________________________________________________________________
2345Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2346{
2347// Start remote grid analysis.
3bdcb562 2348 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2349 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2350 if (!mgr || !mgr->IsInitialized()) {
2351 Error("StartAnalysis", "You need an initialized analysis manager for this");
2352 return kFALSE;
2353 }
2354 // Are we in PROOF mode ?
2355 if (mgr->IsProofMode()) {
90d50a8c 2356 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2357 if (fProofCluster.IsNull()) {
2358 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2359 return kFALSE;
2360 }
2361 if (fProofDataSet.IsNull() && !testMode) {
2362 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2363 return kFALSE;
2364 }
2365 // Set the needed environment
2366 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2367 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2368 if (fProofReset && !testMode) {
2369 if (fProofReset==1) {
2370 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2371 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2372 } else {
2373 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2374 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2375 }
2376 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2377 return kFALSE;
2378 }
2379 // Do we need to change the ROOT version ? The success of this cannot be checked.
2380 if (!fRootVersionForProof.IsNull() && !testMode) {
2381 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2382 fProofCluster.Data(), fRootVersionForProof.Data()));
2383 }
2384 // Connect to PROOF and check the status
2385 Long_t proof = 0;
27734f0e 2386 TString sworkers;
2387 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2388 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2389 if (!testMode) {
27734f0e 2390 if (!sworkers.IsNull())
2391 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2392 else
2393 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2394 } else {
2395 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2396 if (!proof) {
2397 Error("StartAnalysis", "Could not start PROOF in test mode");
2398 return kFALSE;
2399 }
2400 }
2401 if (!proof) {
2402 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2403 return kFALSE;
2404 }
27734f0e 2405 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2406 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2407 // Is dataset existing ?
2408 if (!testMode) {
2409 TString dataset = fProofDataSet;
2410 Int_t index = dataset.Index("#");
2411 if (index>=0) dataset.Remove(index);
830acc4c 2412// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2413// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2414// return kFALSE;
2415// }
2416// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2417 }
2418 // Is ClearPackages() needed ?
2419 if (TestSpecialBit(kClearPackages)) {
2420 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2421 gROOT->ProcessLine("gProof->ClearPackages();");
2422 }
2423 // Is a given aliroot mode requested ?
2424 TList optionsList;
1ac846e9 2425 TString parLibs;
830acc4c 2426 if (!fAliRootMode.IsNull()) {
3bdcb562 2427 TString alirootMode = fAliRootMode;
2428 if (alirootMode == "default") alirootMode = "";
2429 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2430 optionsList.SetOwner();
2431 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2432 // Check the additional libs to be loaded
2433 TString extraLibs;
1ac846e9 2434 Bool_t parMode = kFALSE;
3bdcb562 2435 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2436 // Parse the extra libs for .so
2437 if (fAdditionalLibs.Length()) {
2438 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2439 TIter next(list);
2440 TObjString *str;
1ac846e9 2441 while((str=(TObjString*)next())) {
2442 if (str->GetString().Contains(".so")) {
2443 if (parMode) {
2444 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2445 break;
2446 }
2447 TString stmp = str->GetName();
2448 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2449 stmp.ReplaceAll(".so","");
2450 if (!extraLibs.IsNull()) extraLibs += ":";
2451 extraLibs += stmp;
2452 continue;
2453 }
2454 if (str->GetString().Contains(".par")) {
2455 // The first par file found in the list will not allow any further .so
2456 parMode = kTRUE;
2457 if (!parLibs.IsNull()) parLibs += ":";
2458 parLibs += str->GetName();
2459 continue;
2460 }
3bdcb562 2461 }
2462 if (list) delete list;
2463 }
2464 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2465 // Check extra includes
2466 if (!fIncludePath.IsNull()) {
2467 TString includePath = fIncludePath;
2468 includePath.ReplaceAll(" ",":");
8ddad121 2469 includePath.ReplaceAll("$ALICE_ROOT","");
2470 includePath.ReplaceAll("${ALICE_ROOT}","");
2471 includePath.ReplaceAll("-I","");
830acc4c 2472 includePath.Strip(TString::kTrailing, ':');
2473 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2474 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2475 }
27734f0e 2476 // Check if connection to grid is requested
2477 if (TestSpecialBit(kProofConnectGrid))
2478 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2479 // Enable AliRoot par
2480 if (testMode) {
2481 // Enable proof lite package
2482 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2483 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2484 TNamed *obj = (TNamed*)optionsList.At(i);
2485 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2486 }
2487 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2488 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2489 Info("StartAnalysis", "AliRootProofLite enabled");
2490 } else {
2491 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2492 return kFALSE;
2493 }
2494 } else {
8ddad121 2495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2496 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2497 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2498 return kFALSE;
2499 }
27734f0e 2500 }
1ac846e9 2501 // Enable first par files from fAdditionalLibs
2502 if (!parLibs.IsNull()) {
2503 TObjArray *list = parLibs.Tokenize(":");
2504 TIter next(list);
2505 TObjString *package;
2506 while((package=(TObjString*)next())) {
40501bb7 2507 TString spkg = package->GetName();
2508 spkg.ReplaceAll(".par", "");
2509 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2510 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2511 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2512 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2513 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2514 return kFALSE;
2515 }
2516 } else {
2517 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2518 return kFALSE;
2519 }
2520 }
2521 if (list) delete list;
2522 }
3bdcb562 2523 } else {
2524 if (fAdditionalLibs.Contains(".so") && !testMode) {
2525 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2526 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2527 return kFALSE;
2528 }
2529 }
2530 // Enable par files if requested
2531 if (fPackages && fPackages->GetEntries()) {
2532 TIter next(fPackages);
2533 TObject *package;
2534 while ((package=next())) {
1ac846e9 2535 // Skip packages already enabled
2536 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2537 TString spkg = package->GetName();
2538 spkg.ReplaceAll(".par", "");
2539 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2540 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2541 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2542 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2543 return kFALSE;
2544 }
2545 } else {
2546 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2547 return kFALSE;
2548 }
2549 }
2550 }
2551 // Do we need to load analysis source files ?
2552 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2553 if (fAnalysisSource.Length()) {
2554 TObjArray *list = fAnalysisSource.Tokenize(" ");
2555 TIter next(list);
2556 TObjString *str;
2557 while((str=(TObjString*)next())) {
2558 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2559 }
2560 if (list) delete list;
2561 }
830acc4c 2562 if (testMode) {
2563 // Register dataset to proof lite.
2564 if (fFileForTestMode.IsNull()) {
2565 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2566 return kFALSE;
2567 }
2568 if (gSystem->AccessPathName(fFileForTestMode)) {
2569 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2570 return kFALSE;
2571 }
2572 TFileCollection *coll = new TFileCollection();
2573 coll->AddFromFile(fFileForTestMode);
8ddad121 2574 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2575 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2576 }
3bdcb562 2577 return kTRUE;
2578 }
c57f56b7 2579
43da816a 2580 // Check if output files have to be taken from the analysis manager
2581 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2582 // Add output files and AOD files
2583 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2584 // Add extra files registered to the analysis manager
f790bc1b 2585 TString extra = GetListOfFiles("ext");
2586 if (!extra.IsNull()) {
149d288c 2587 extra.ReplaceAll(".root", "*.root");
f790bc1b 2588 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2589 fOutputFiles += extra;
84fcd93f 2590 }
149d288c 2591 // Compose the output archive.
b83f84f0 2592 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2593 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2594 }
f7b1cbc2 2595// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2596 if (TestBit(AliAnalysisGrid::kOffline)) {
2597 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2598 \n there nor any job run. You can revise the JDL and analysis \
2599 \n macro then run the same in \"submit\" mode.");
2600 } else if (TestBit(AliAnalysisGrid::kTest)) {
2601 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2602 \n dataset.");
2603 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2604 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2605 \n space and job submitted.");
2606 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2607 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2608 if (fMergeViaJDL) CheckInputData();
5513444a 2609 return kTRUE;
c57f56b7 2610 } else {
2611 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2612 }
2613
348be253 2614 Print();
c57f56b7 2615 if (!Connect()) {
2616 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2617 return kFALSE;
16a4353c 2618 }
d3b18c4c 2619 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2620 if (!CheckInputData()) {
2621 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2622 return kFALSE;
c57f56b7 2623 }
d3339be3 2624 if (!CreateDataset(fDataPattern)) {
2625 TString serror;
2626 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2627 if (fRunNumbers.Length()) serror = "run numbers";
2628 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2629 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2630 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2631 return kFALSE;
2632 }
c57f56b7 2633 WriteAnalysisFile();
2634 WriteAnalysisMacro();
2635 WriteExecutable();
2636 WriteValidationScript();
0f389141 2637 if (fMergeViaJDL) {
2638 WriteMergingMacro();
2639 WriteMergeExecutable();
2640 WriteValidationScript(kTRUE);
2641 }
5513444a 2642 if (!CreateJDL()) return kFALSE;
2643 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2644 if (testMode) {
c57f56b7 2645 // Locally testing the analysis
2646 Info("StartAnalysis", "\n_______________________________________________________________________ \
2647 \n Running analysis script in a daughter shell as on a worker node \
2648 \n_______________________________________________________________________");
149d288c 2649 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2650 TIter next(list);
2651 TObjString *str;
a2f5fc01 2652 TString outputFile;
c57f56b7 2653 while((str=(TObjString*)next())) {
a2f5fc01 2654 outputFile = str->GetString();
2655 Int_t index = outputFile.Index("@");
2656 if (index > 0) outputFile.Remove(index);
2657 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2658 }
2659 delete list;
2660 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2661 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2662// gSystem->Exec("cat stdout");
5513444a 2663 return kFALSE;
c57f56b7 2664 }
5513444a 2665 // Check if submitting is managed by LPM manager
a3e84053 2666 if (fProductionMode) {
5513444a 2667 TString prodfile = fJDLName;
2668 prodfile.ReplaceAll(".jdl", ".prod");
2669 WriteProductionFile(prodfile);
2670 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2671 return kFALSE;
2672 }
a8739e8a 2673 // Submit AliEn job(s)
d2a409b2 2674 gGrid->Cd(fGridOutputDir);
a8739e8a 2675 TGridResult *res;
c57f56b7 2676 TString jobID = "";
d2a409b2 2677 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2678 // Submit a given xml or a set of runs
a8739e8a 2679 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2680 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2681 if (res) {
2682 const char *cjobId = res->GetKey(0,"jobId");
2683 if (!cjobId) {
a03be957 2684 gGrid->Stdout();
2685 gGrid->Stderr();
a8739e8a 2686 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2687 return kFALSE;
a8739e8a 2688 } else {
2689 Info("StartAnalysis", "\n_______________________________________________________________________ \
2690 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2691 \n_______________________________________________________________________",
2692 fJDLName.Data(), cjobId);
2693 jobID = cjobId;
2694 }
2695 delete res;
a03be957 2696 } else {
2697 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2698 return kFALSE;
a8739e8a 2699 }
2700 } else {
d2a409b2 2701 // Submit for a range of enumeration of runs.
a03be957 2702 if (!Submit()) return kFALSE;
c57f56b7 2703 }
a8739e8a 2704
c57f56b7 2705 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2706 \n You may exit at any time and terminate the job later using the option <terminate> \
2707 \n ##################################################################################", jobID.Data());
bb885a9e 2708 gSystem->Exec("aliensh");
5513444a 2709 return kTRUE;
c57f56b7 2710}
2711
f790bc1b 2712//______________________________________________________________________________
2713const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2714{
2715// Get a comma-separated list of output files of the requested type.
2716// Type can be (case unsensitive):
2717// aod - list of aod files (std, extensions and filters)
2718// out - list of output files connected to containers (but not aod's or extras)
2719// ext - list of extra files registered to the manager
2720// ter - list of files produced in terminate
2721 static TString files;
2722 files = "";
2723 TString stype = type;
2724 stype.ToLower();
2725 TString aodfiles, extra;
2726 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2727 if (!mgr) {
2728 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2729 return files.Data();
2730 }
2731 if (mgr->GetOutputEventHandler()) {
2732 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2733 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2734 if (!extraaod.IsNull()) {
2735 aodfiles += ",";
2736 aodfiles += extraaod;
2737 }
2738 }
2739 if (stype.Contains("aod")) {
2740 files = aodfiles;
2741 if (stype == "aod") return files.Data();
2742 }
2743 // Add output files that are not in the list of AOD files
2744 TString outputfiles = "";
2745 TIter next(mgr->GetOutputs());
2746 AliAnalysisDataContainer *output;
2747 const char *filename = 0;
2748 while ((output=(AliAnalysisDataContainer*)next())) {
2749 filename = output->GetFileName();
2750 if (!(strcmp(filename, "default"))) continue;
2751 if (outputfiles.Contains(filename)) continue;
2752 if (aodfiles.Contains(filename)) continue;
2753 if (!outputfiles.IsNull()) outputfiles += ",";
2754 outputfiles += filename;
2755 }
2756 if (stype.Contains("out")) {
2757 if (!files.IsNull()) files += ",";
2758 files += outputfiles;
2759 if (stype == "out") return files.Data();
2760 }
2761 // Add extra files registered to the analysis manager
2762 TString sextra;
2763 extra = mgr->GetExtraFiles();
2764 if (!extra.IsNull()) {
2765 extra.Strip();
2766 extra.ReplaceAll(" ", ",");
2767 TObjArray *fextra = extra.Tokenize(",");
2768 TIter nextx(fextra);
2769 TObject *obj;
2770 while ((obj=nextx())) {
2771 if (aodfiles.Contains(obj->GetName())) continue;
2772 if (outputfiles.Contains(obj->GetName())) continue;
2773 if (sextra.Contains(obj->GetName())) continue;
2774 if (!sextra.IsNull()) sextra += ",";
2775 sextra += obj->GetName();
2776 }
2777 delete fextra;
2778 if (stype.Contains("ext")) {
2779 if (!files.IsNull()) files += ",";
2780 files += sextra;
2781 }
2782 }
2783 if (stype == "ext") return files.Data();
2784 TString termfiles;
2785 if (!fTerminateFiles.IsNull()) {
2786 fTerminateFiles.Strip();
2787 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2788 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2789 TIter nextx(fextra);
2790 TObject *obj;
2791 while ((obj=nextx())) {
2792 if (aodfiles.Contains(obj->GetName())) continue;
2793 if (outputfiles.Contains(obj->GetName())) continue;
2794 if (termfiles.Contains(obj->GetName())) continue;
2795 if (sextra.Contains(obj->GetName())) continue;
2796 if (!termfiles.IsNull()) termfiles += ",";
2797 termfiles += obj->GetName();
2798 }
2799 delete fextra;
2800 }
2801 if (stype.Contains("ter")) {
2802 if (!files.IsNull() && !termfiles.IsNull()) {
2803 files += ",";
2804 files += termfiles;
2805 }
2806 }
2807 return files.Data();
2808}
2809
d2a409b2 2810//______________________________________________________________________________
a03be957 2811Bool_t AliAnalysisAlien::Submit()
d2a409b2 2812{
2813// Submit all master jobs.
2814 Int_t nmasterjobs = fInputFiles->GetEntries();
2815 Long_t tshoot = gSystem->Now();
a03be957 2816 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2817 while (fNsubmitted < nmasterjobs) {
2818 Long_t now = gSystem->Now();
2819 if ((now-tshoot)>30000) {
2820 tshoot = now;
a03be957 2821 if (!SubmitNext()) return kFALSE;
d2a409b2 2822 }
2823 }
a03be957 2824 return kTRUE;
d2a409b2 2825}
2826
0f389141 2827//______________________________________________________________________________
a03be957 2828Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2829{
2830// Submit all merging jobs.
2831 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2832 gGrid->Cd(fGridOutputDir);
2833 TString mergeJDLName = fExecutable;
2834 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2835 if (!fInputFiles) {
2836 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2837 return kFALSE;
2838 }
0f389141 2839 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2840 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2841 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2842 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2843 if (fOutputToRunNo) {
2844 // The output directory is the run number
2845 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2846 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2847 } else {
17b4e9a4 2848 if (!fRunNumbers.Length() && !fRunRange[0]) {
2849 // The output directory is the grid outdir
2850 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2851 runOutDir = fGridOutputDir;
2852 } else {
2853 // The output directory is the master number in 3 digits format
2854 printf("### Submitting merging job for master <%03d>\n", i);
2855 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2856 }
7c2cd90a 2857 }
2858 // Check now the number of merging stages.
2b222f7c 2859 TObjArray *list = fOutputFiles.Tokenize(",");
2860 TIter next(list);
2861 TObjString *str;
2862 TString outputFile;
2863 while((str=(TObjString*)next())) {
2864 outputFile = str->GetString();
2865 Int_t index = outputFile.Index("@");
2866 if (index > 0) outputFile.Remove(index);
2867 if (!fMergeExcludes.Contains(outputFile)) break;
2868 }
2869 delete list;
b3e07543 2870 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2871 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 2872 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 2873 }
a03be957 2874 if (!ntosubmit) return kTRUE;
9f020952 2875 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2876 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2877 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2878 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2879 \n ################################################################################################################");
0f389141 2880 gSystem->Exec("aliensh");
a03be957 2881 return kTRUE;
0f389141 2882}
2883
d2a409b2 2884//______________________________________________________________________________
a03be957 2885Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2886{
08d5b699 2887// Submit next bunch of master jobs if the queue is free. The first master job is
2888// submitted right away, while the next will not be unless the previous was split.
2889// The plugin will not submit new master jobs if there are more that 500 jobs in
2890// waiting phase.
d2a409b2 2891 static Bool_t iscalled = kFALSE;
2892 static Int_t firstmaster = 0;
2893 static Int_t lastmaster = 0;
2894 static Int_t npermaster = 0;
a03be957 2895 if (iscalled) return kTRUE;
d2a409b2 2896 iscalled = kTRUE;
2897 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2898 Int_t ntosubmit = 0;
2899 TGridResult *res;
2900 TString jobID = "";
3bdcb562 2901 Int_t nmasterjobs = fInputFiles->GetEntries();
2902 if (!fNsubmitted) {
2903 ntosubmit = 1;
2904 if (!IsUseSubmitPolicy()) {
90d50a8c 2905 if (nmasterjobs>5)
3bdcb562 2906 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2907 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2908 ntosubmit = nmasterjobs;
2909 }
2910 } else {
d2a409b2 2911 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2912 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2913 // If last master not split, just return
a03be957 2914 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2915 // No more than 100 waiting jobs
08d5b699 2916 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2917 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2918 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2919 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2920 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2921 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2922 }
d2a409b2 2923 for (Int_t i=0; i<ntosubmit; i++) {
2924 // Submit for a range of enumeration of runs.
a03be957 2925 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2926 TString query;
cd11251e 2927 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2928 runOutDir.ReplaceAll(".xml", "");
2929 if (fOutputToRunNo)
2930 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2931 else
2932 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2933 printf("********* %s\n",query.Data());
d2a409b2 2934 res = gGrid->Command(query);
2935 if (res) {
98ca124f 2936 TString cjobId1 = res->GetKey(0,"jobId");
2937 if (!cjobId1.Length()) {
d2a409b2 2938 iscalled = kFALSE;
a03be957 2939 gGrid->Stdout();
2940 gGrid->Stderr();
2941 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2942 return kFALSE;
d2a409b2 2943 } else {
2944 Info("StartAnalysis", "\n_______________________________________________________________________ \
2945 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2946 \n_______________________________________________________________________",
98ca124f 2947 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2948 jobID += cjobId1;
2949 jobID += " ";
98ca124f 2950 lastmaster = cjobId1.Atoi();
d2a409b2 2951 if (!firstmaster) firstmaster = lastmaster;
2952 fNsubmitted++;
2953 }
2954 delete res;
a03be957 2955 } else {
2956 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2957 return kFALSE;
d2a409b2 2958 }
2959 }
2960 iscalled = kFALSE;
a03be957 2961 return kTRUE;
d2a409b2 2962}
2963
c57f56b7 2964//______________________________________________________________________________
2965void AliAnalysisAlien::WriteAnalysisFile()
2966{
f10e8481 2967// Write current analysis manager into the file <analysisFile>
2968 TString analysisFile = fExecutable;
2969 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2970 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2971 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2972 if (!mgr || !mgr->IsInitialized()) {
2973 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2974 return;
2975 }
2976 // Check analysis type
2977 TObject *handler;
2978 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2979 handler = (TObject*)mgr->GetInputEventHandler();
2980 if (handler) {
2981 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2982 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2983 }
2984 TDirectory *cdir = gDirectory;
f10e8481 2985 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2986 if (file) {
e16a394c 2987 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2988 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2989 // Unless merging makes no sense
2990 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2991 mgr->Write();
2992 delete file;
fe2d7fc2 2993 // Enable termination for local jobs
2994 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2995 }
2996 if (cdir) cdir->cd();
f10e8481 2997 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2998 }
2999 Bool_t copy = kTRUE;
d3b18c4c 3000 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3001 if (copy) {
3002 CdWork();
3003 TString workdir = gGrid->GetHomeDirectory();
3004 workdir += fGridWorkingDir;
d3b18c4c 3005 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3006 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3007 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3008 }
3009}
3010
3011//______________________________________________________________________________
3012void AliAnalysisAlien::WriteAnalysisMacro()
3013{
3014// Write the analysis macro that will steer the analysis in grid mode.
3015 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3016 ofstream out;
3017 out.open(fAnalysisMacro.Data(), ios::out);
3018 if (!out.good()) {
3019 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3020 return;
3021 }
5b273635 3022 Bool_t hasSTEERBase = kFALSE;
3023 Bool_t hasESD = kFALSE;
3024 Bool_t hasAOD = kFALSE;
3025 Bool_t hasANALYSIS = kFALSE;
3026 Bool_t hasANALYSISalice = kFALSE;
3027 Bool_t hasCORRFW = kFALSE;
c57f56b7 3028 TString func = fAnalysisMacro;
3029 TString type = "ESD";
3030 TString comment = "// Analysis using ";
c85cfc0f 3031 if (IsUseMCchain()) {
3032 type = "MC";
3033 comment += "MC";
3034 } else {
3035 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3036 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3037 type = "AOD";
3038 comment += "AOD";
3039 }
3040 }
0df6ccf2 3041 if (type!="AOD" && fFriendChainName!="") {
3042 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3043 return;
3044 }
c57f56b7 3045 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3046 else comment += " data";
3047 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3048 func.ReplaceAll(".C", "");
3049 out << "void " << func.Data() << "()" << endl;
3050 out << "{" << endl;
3051 out << comment.Data() << endl;
3052 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3053 out << " TStopwatch timer;" << endl;
3054 out << " timer.Start();" << endl << endl;
13f0c00b 3055 // Change temp directory to current one
3056 out << "// Set temporary merging directory to current one" << endl;
3057 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
b3e07543 3058 // Reset existing include path
3059 out << "// Reset existing include path and add current directory first in the search" << endl;
3060 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3061 if (!fExecutableCommand.Contains("aliroot")) {
3062 out << "// load base root libraries" << endl;
3063 out << " gSystem->Load(\"libTree\");" << endl;
3064 out << " gSystem->Load(\"libGeom\");" << endl;
3065 out << " gSystem->Load(\"libVMC\");" << endl;
3066 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3067 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3068 }
d5c6455a 3069 if (fAdditionalRootLibs.Length()) {
47a4137d 3070 // in principle libtree /lib geom libvmc etc. can go into this list, too
3071 out << "// Add aditional libraries" << endl;
3072 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3073 TIter next(list);
3074 TObjString *str;
3075 while((str=(TObjString*)next())) {
3076 if (str->GetString().Contains(".so"))
3077 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3078 }
47a4137d 3079 if (list) delete list;
d5c6455a 3080 }
57377eb5 3081 out << "// Load analysis framework libraries" << endl;
652561d5 3082 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3083 if (!fPackages) {
f790bc1b 3084 if (!fExecutableCommand.Contains("aliroot")) {
3085 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3086 out << " gSystem->Load(\"libESD\");" << endl;
3087 out << " gSystem->Load(\"libAOD\");" << endl;
3088 }
4e5c5506 3089 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3090 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3091 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3092 } else {
4e5c5506 3093 TIter next(fPackages);
3094 TObject *obj;
57377eb5 3095 TString pkgname;
57377eb5 3096 while ((obj=next())) {
3097 pkgname = obj->GetName();
4478e6f1 3098 if (pkgname == "STEERBase" ||
3099 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3100 if (pkgname == "ESD" ||
3101 pkgname == "ESD.par") hasESD = kTRUE;
3102 if (pkgname == "AOD" ||
3103 pkgname == "AOD.par") hasAOD = kTRUE;
3104 if (pkgname == "ANALYSIS" ||
3105 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3106 if (pkgname == "ANALYSISalice" ||
3107 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3108 if (pkgname == "CORRFW" ||
3109 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3110 }
3111 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3112 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3113 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3114 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3115 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3116 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3117 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3118 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3119 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3120 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3121 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3122 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3123 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3124 out << "// Compile other par packages" << endl;
3125 next.Reset();
fcc9bb6f 3126 while ((obj=next())) {
3127 pkgname = obj->GetName();
4478e6f1 3128 if (pkgname == "STEERBase" ||
3129 pkgname == "STEERBase.par" ||
3130 pkgname == "ESD" ||
3131 pkgname == "ESD.par" ||
3132 pkgname == "AOD" ||
3133 pkgname == "AOD.par" ||
3134 pkgname == "ANALYSIS" ||
3135 pkgname == "ANALYSIS.par" ||
3136 pkgname == "ANALYSISalice" ||
3137 pkgname == "ANALYSISalice.par" ||
3138 pkgname == "CORRFW" ||
3139 pkgname == "CORRFW.par") continue;
5b273635 3140 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3141 }
4e5c5506 3142 }
b3e07543 3143 out << "// include path" << endl;
3144 // Get the include path from the interpreter and remove entries pointing to AliRoot
3145 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3146 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3147 out << " TIter nextpath(listpaths);" << endl;
3148 out << " TObjString *pname;" << endl;
3149 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3150 out << " TString current = pname->GetName();" << endl;
3151 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3152 out << " gSystem->AddIncludePath(current);" << endl;
3153 out << " }" << endl;
3154 out << " if (listpaths) delete listpaths;" << endl;
3155 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3156 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3157 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3158 if (fAdditionalLibs.Length()) {
3159 out << "// Add aditional AliRoot libraries" << endl;
3160 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3161 TIter next(list);
3162 TObjString *str;
3163 while((str=(TObjString*)next())) {
3164 if (str->GetString().Contains(".so"))
3165 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3166 if (str->GetString().Contains(".par"))
3167 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3168 }
3169 if (list) delete list;
3170 }
3171 out << endl;
c57f56b7 3172 out << "// analysis source to be compiled at runtime (if any)" << endl;
3173 if (fAnalysisSource.Length()) {
3174 TObjArray *list = fAnalysisSource.Tokenize(" ");
3175 TIter next(list);
3176 TObjString *str;
3177 while((str=(TObjString*)next())) {
3178 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3179 }
3180 if (list) delete list;
3181 }
3182 out << endl;
b3e07543 3183// out << " printf(\"Currently load libraries:\\n\");" << endl;
3184// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3185 if (fFastReadOption) {
d3339be3 3186 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3187 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3188 out << "// fast xrootd reading enabled" << endl;
3189 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3190 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3191 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3192 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3193 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3194 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3195 }
c57f56b7 3196 out << "// connect to AliEn and make the chain" << endl;
3197 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3198 out << "// read the analysis manager from file" << endl;
f10e8481 3199 TString analysisFile = fExecutable;
3200 analysisFile.ReplaceAll(".sh", ".root");
3201 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3202 out << " if (!file) return;" << endl;
3203 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3204 out << " AliAnalysisManager *mgr = 0;" << endl;
3205 out << " TKey *key;" << endl;
3206 out << " while ((key=(TKey*)nextkey())) {" << endl;
3207 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3208 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3209 out << " };" << endl;
3210 out << " if (!mgr) {" << endl;
205b201f 3211 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3212 out << " return;" << endl;
3213 out << " }" << endl << endl;
3214 out << " mgr->PrintStatus();" << endl;
52b6a92b 3215 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3216 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3217 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3218 } else {
9a8288ff 3219 if (TestBit(AliAnalysisGrid::kTest))
3220 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3221 else
3222 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3223 }
3224 }
242accb2 3225 if (IsUsingTags()) {
3226 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3227 } else {
3228 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3229 }
c57f56b7 3230 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3231 out << " timer.Stop();" << endl;
3232 out << " timer.Print();" << endl;
c57f56b7 3233 out << "}" << endl << endl;
3234 if (IsUsingTags()) {
3235 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3236 out << "{" << endl;
3237 out << "// Create a chain using tags from the xml file." << endl;
3238 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3239 out << " if (!coll) {" << endl;
3240 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3241 out << " return NULL;" << endl;
3242 out << " }" << endl;
3243 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3244 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3245 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3246 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3247 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3248 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3249 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3250 out << " // Check if the cuts configuration file was provided" << endl;
3251 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3252 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3253 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3254 out << " }" << endl;
0df6ccf2 3255 if (fFriendChainName=="") {
3256 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3257 } else {
3258 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3259 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3260 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3261 }
c57f56b7 3262 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3263 out << " chain->ls();" << endl;
3264 out << " return chain;" << endl;
fcc9bb6f 3265 out << "}" << endl << endl;
c57f56b7 3266 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3267 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3268 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3269 msg += " AliLHCTagCuts *lhcCuts,\n";
3270 msg += " AliDetectorTagCuts *detCuts,\n";
3271 msg += " AliEventTagCuts *evCuts)";
23329835 3272 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3273 }
0df6ccf2 3274 }
3275 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3276 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3277 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3278 out << "{" << endl;
3279 out << "// Create a chain using url's from xml file" << endl;
242accb2 3280 out << " TString filename;" << endl;
3281 out << " Int_t run = 0;" << endl;
c85cfc0f 3282 if (IsUseMCchain()) {
3283 out << " TString treename = \"TE\";" << endl;
3284 } else {
3285 out << " TString treename = type;" << endl;
3286 out << " treename.ToLower();" << endl;
3287 out << " treename += \"Tree\";" << endl;
3288 }
e02fee64 3289 out << " printf(\"***************************************\\n\");" << endl;
3290 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3291 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3292 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3293 out << " if (!coll) {" << endl;
3294 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3295 out << " return NULL;" << endl;
3296 out << " }" << endl;
242accb2 3297 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3298 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3299 if(fFriendChainName!="") {
3300 out << " TChain *chainFriend = new TChain(treename);" << endl;
3301 }
c57f56b7 3302 out << " coll->Reset();" << endl;
0df6ccf2 3303 out << " while (coll->Next()) {" << endl;
242accb2 3304 out << " filename = coll->GetTURL("");" << endl;
3305 out << " if (mgr) {" << endl;
3306 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3307 out << " if (nrun && nrun != run) {" << endl;
3308 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3309 out << " mgr->SetRunFromPath(nrun);" << endl;
3310 out << " run = nrun;" << endl;
3311 out << " }" << endl;
3312 out << " }" << endl;
3313 out << " chain->Add(filename);" << endl;
0df6ccf2 3314 if(fFriendChainName!="") {
3315 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3316 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3317 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3318 out << " chainFriend->Add(fileFriend.Data());" << endl;
3319 }
3320 out << " }" << endl;
c57f56b7 3321 out << " if (!chain->GetNtrees()) {" << endl;
3322 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3323 out << " return NULL;" << endl;
3324 out << " }" << endl;
0df6ccf2 3325 if(fFriendChainName!="") {
3326 out << " chain->AddFriend(chainFriend);" << endl;
3327 }
c57f56b7 3328 out << " return chain;" << endl;
fcc9bb6f 3329 out << "}" << endl << endl;
c57f56b7 3330 }
5b273635 3331 if (hasANALYSISalice) {
3332 out <<"//________________________________________________________________________________" << endl;
3333 out << "Bool_t SetupPar(const char *package) {" << endl;
3334 out << "// Compile the package and set it up." << endl;
3335 out << " TString pkgdir = package;" << endl;
3336 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3337 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3338 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3339 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3340 out << " // Check for BUILD.sh and execute" << endl;
3341 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3342 out << " printf(\"*******************************\\n\");" << endl;
3343 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3344 out << " printf(\"*******************************\\n\");" << endl;
3345 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3346 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3347 out << " gSystem->ChangeDirectory(cdir);" << endl;
3348 out << " return kFALSE;" << endl;
3349 out << " }" << endl;
3350 out << " } else {" << endl;
3351 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3352 out << " gSystem->ChangeDirectory(cdir);" << endl;
3353 out << " return kFALSE;" << endl;
3354 out << " }" << endl;
3355 out << " // Check for SETUP.C and execute" << endl;
3356 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3357 out << " printf(\"*******************************\\n\");" << endl;
3358 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3359 out << " printf(\"*******************************\\n\");" << endl;
3360 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3361 out << " } else {" << endl;
3362 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3363 out << " gSystem->ChangeDirectory(cdir);" << endl;
3364 out << " return kFALSE;" << endl;
3365 out << " }" << endl;
3366 out << " // Restore original workdir" << endl;
3367 out << " gSystem->ChangeDirectory(cdir);" << endl;
3368 out << " return kTRUE;" << endl;
3369 out << "}" << endl;
3370 }
c57f56b7 3371 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3372 }
3373 Bool_t copy = kTRUE;
d3b18c4c 3374 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3375 if (copy) {
3376 CdWork();
3377 TString workdir = gGrid->GetHomeDirectory();
3378 workdir += fGridWorkingDir;
3379 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3380 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3381 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3382 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3383 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3384 }
3385 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3386 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3387 }
3388}
3389
0f389141 3390//______________________________________________________________________________
3391void AliAnalysisAlien::WriteMergingMacro()
3392{
3393// Write a macro to merge the outputs per master job.
3394 if (!fMergeViaJDL) return;
3395 if (!fOutputFiles.Length()) {
3396 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3397 return;
3398 }
3399 TString mergingMacro = fExecutable;
3400 mergingMacro.ReplaceAll(".sh","_merge.C");
3401 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3402 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3403 ofstream out;
3404 out.open(mergingMacro.Data(), ios::out);
3405 if (!out.good()) {
3406 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3407 return;
3408 }
5b273635 3409 Bool_t hasSTEERBase = kFALSE;
3410 Bool_t hasESD = kFALSE;
3411 Bool_t hasAOD = kFALSE;
3412 Bool_t hasANALYSIS = kFALSE;
3413 Bool_t hasANALYSISalice = kFALSE;
3414 Bool_t hasCORRFW = kFALSE;
0f389141 3415 TString func = mergingMacro;
3416 TString comment;
3417 func.ReplaceAll(".C", "");
f47d5cb4 3418 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3419 out << "{" << endl;
3420 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3421 out << " TStopwatch timer;" << endl;
3422 out << " timer.Start();" << endl << endl;
b3e07543 3423 // Reset existing include path
3424 out << "// Reset existing include path and add current directory first in the search" << endl;
3425 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3426 if (!fExecutableCommand.Contains("aliroot")) {
3427 out << "// load base root libraries" << endl;
3428 out << " gSystem->Load(\"libTree\");" << endl;
3429 out << " gSystem->Load(\"libGeom\");" << endl;
3430 out << " gSystem->Load(\"libVMC\");" << endl;
3431 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3432 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3433 }
0f389141 3434 if (fAdditionalRootLibs.Length()) {
3435 // in principle libtree /lib geom libvmc etc. can go into this list, too
3436 out << "// Add aditional libraries" << endl;
3437 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3438 TIter next(list);
3439 TObjString *str;
3440 while((str=(TObjString*)next())) {
3441 if (str->GetString().Contains(".so"))
3442 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3443 }
3444 if (list) delete list;
3445 }
0f389141 3446 out << "// Load analysis framework libraries" << endl;
3447 if (!fPackages) {
7c2cd90a 3448 if (!fExecutableCommand.Contains("aliroot")) {
3449 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3450 out << " gSystem->Load(\"libESD\");" << endl;
3451 out << " gSystem->Load(\"libAOD\");" << endl;
3452 }
0f389141 3453 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3454 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3455 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3456 } else {
3457 TIter next(fPackages);
3458 TObject *obj;
3459 TString pkgname;
5b273635 3460 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3461 while ((obj=next())) {
3462 pkgname = obj->GetName();
3463 if (pkgname == "STEERBase" ||
3464 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3465 if (pkgname == "ESD" ||
3466 pkgname == "ESD.par") hasESD = kTRUE;
3467 if (pkgname == "AOD" ||
3468 pkgname == "AOD.par") hasAOD = kTRUE;
3469 if (pkgname == "ANALYSIS" ||
3470 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3471 if (pkgname == "ANALYSISalice" ||
3472 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3473 if (pkgname == "CORRFW" ||
3474 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3475 }
5b273635 3476 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3477 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3478 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3479 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3480 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3481 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3482 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3483 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3484 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3485 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3486 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3487 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3488 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3489 out << "// Compile other par packages" << endl;
3490 next.Reset();
3491 while ((obj=next())) {
3492 pkgname = obj->GetName();
3493 if (pkgname == "STEERBase" ||
3494 pkgname == "STEERBase.par" ||
3495 pkgname == "ESD" ||
3496 pkgname == "ESD.par" ||
3497 pkgname == "AOD" ||
3498 pkgname == "AOD.par" ||
3499 pkgname == "ANALYSIS" ||
3500 pkgname == "ANALYSIS.par" ||
3501 pkgname == "ANALYSISalice" ||
3502 pkgname == "ANALYSISalice.par" ||
3503 pkgname == "CORRFW" ||
3504 pkgname == "CORRFW.par") continue;
5b273635 3505 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3506 }
3507 }
b3e07543 3508 out << "// include path" << endl;
3509 // Get the include path from the interpreter and remove entries pointing to AliRoot
3510 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3511 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3512 out << " TIter nextpath(listpaths);" << endl;
3513 out << " TObjString *pname;" << endl;
3514 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3515 out << " TString current = pname->GetName();" << endl;
3516 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3517 out << " gSystem->AddIncludePath(current);" << endl;
3518 out << " }" << endl;
3519 out << " if (listpaths) delete listpaths;" << endl;
3520 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3521 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3522 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3523 if (fAdditionalLibs.Length()) {
3524 out << "// Add aditional AliRoot libraries" << endl;
3525 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3526 TIter next(list);
3527 TObjString *str;
3528 while((str=(TObjString*)next())) {
3529 if (str->GetString().Contains(".so"))
3530 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3531 }
3532 if (list) delete list;
3533 }
3534 out << endl;
3535 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3536 if (fAnalysisSource.Length()) {
3537 TObjArray *list = fAnalysisSource.Tokenize(" ");
3538 TIter next(list);
3539 TObjString *str;
3540 while((str=(TObjString*)next())) {
3541 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3542 }
3543 if (list) delete list;
3544 }
149d288c 3545 out << endl;
3546
0f389141 3547 if (fFastReadOption) {
3548 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3549 out << "// fast xrootd reading enabled" << endl;
3550 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3551 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3552 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3553 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3554 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3555 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3556 }
3557 // Change temp directory to current one
3558 out << "// Set temporary merging directory to current one" << endl;
3559 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3560 out << "// Connect to AliEn" << endl;
3561 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3562 out << " TString outputDir = dir;" << endl;
f790bc1b 3563 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3564 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3565 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3566 out << " TIter *iter = new TIter(list);" << endl;
3567 out << " TObjString *str;" << endl;
a2f5fc01 3568 out << " TString outputFile;" << endl;
0f389141 3569 out << " Bool_t merged = kTRUE;" << endl;
3570 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3571 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3572 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3573 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3574 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3575 out << " // Skip already merged outputs" << endl;
a2f5fc01 3576 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3577 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3578 out << " continue;" << endl;
3579 out << " }" << endl;
a2f5fc01 3580 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3581 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3582 out << " if (!merged) {" << endl;
a2f5fc01 3583 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3584 out << " return;" << endl;
0f389141 3585 out << " }" << endl;
3586 out << " }" << endl;
7c2cd90a 3587 out << " // all outputs merged, validate" << endl;
3588 out << " ofstream out;" << endl;
3589 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3590 out << " out.close();" << endl;
3591 out << " // read the analysis manager from file" << endl;
f866cba5 3592 TString analysisFile = fExecutable;
3593 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3594 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3595 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3596 out << " if (!file) return;" << endl;
f866cba5 3597 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3598 out << " AliAnalysisManager *mgr = 0;" << endl;
3599 out << " TKey *key;" << endl;
3600 out << " while ((key=(TKey*)nextkey())) {" << endl;
3601 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3602 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3603 out << " };" << endl;
3604 out << " if (!mgr) {" << endl;
3605 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3606 out << " return;" << endl;
3607 out << " }" << endl << endl;
5c4250fc 3608 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3609 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3610 out << " mgr->PrintStatus();" << endl;
3611 if (AliAnalysisManager::GetAnalysisManager()) {
3612 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3613 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3614 } else {
9a8288ff 3615 if (TestBit(AliAnalysisGrid::kTest))
3616 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3617 else
3618 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3619 }
3620 }
9a8288ff 3621 out << " TTree *tree = NULL;" << endl;
3622 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3623 out << "}" << endl << endl;
5b273635 3624 if (hasANALYSISalice) {
3625 out <<"//________________________________________________________________________________" << endl;
3626 out << "Bool_t SetupPar(const char *package) {" << endl;
3627 out << "// Compile the package and set it up." << endl;
3628 out << " TString pkgdir = package;" << endl;
3629 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3630 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3631 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3632 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3633 out << " // Check for BUILD.sh and execute" << endl;
3634 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3635 out << " printf(\"*******************************\\n\");" << endl;
3636 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3637 out << " printf(\"*******************************\\n\");" << endl;
3638 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3639 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3640 out << " gSystem->ChangeDirectory(cdir);" << endl;
3641 out << " return kFALSE;" << endl;
3642 out << " }" << endl;
3643 out << " } else {" << endl;
3644 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3645 out << " gSystem->ChangeDirectory(cdir);" << endl;
3646 out << " return kFALSE;" << endl;
3647 out << " }" << endl;
3648 out << " // Check for SETUP.C and execute" << endl;
3649 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3650 out << " printf(\"*******************************\\n\");" << endl;
3651 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3652 out << " printf(\"*******************************\\n\");" << endl;
3653 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3654 out << " } else {" << endl;
3655 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3656 out << " gSystem->ChangeDirectory(cdir);" << endl;
3657 out << " return kFALSE;" << endl;
3658 out << " }" << endl;
3659 out << " // Restore original workdir" << endl;
3660 out << " gSystem->ChangeDirectory(cdir);" << endl;
3661 out << " return kTRUE;" << endl;
3662 out << "}" << endl;
3663 }
0f389141 3664 }
3665 Bool_t copy = kTRUE;
d3b18c4c 3666 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3667 if (copy) {
3668 CdWork();
3669 TString workdir = gGrid->GetHomeDirectory();
3670 workdir += fGridWorkingDir;
3671 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3672 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3673 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3674 }
3675}
3676
3677//______________________________________________________________________________
3678Bool_t AliAnalysisAlien::SetupPar(const char *package)
3679{
205b201f 3680// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3681// Note that for loading the compiled library. The current directory should have precedence in
3682// LD_LIBRARY_PATH
3683 TString pkgdir = package;
3684 pkgdir.ReplaceAll(".par","");
b50a41c3 3685 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3686 TString cdir = gSystem->WorkingDirectory();
3687 gSystem->ChangeDirectory(pkgdir);
3688 // Check for BUILD.sh and execute
3689 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3690 printf("**************************************************\n");
3691 printf("*** Building PAR archive %s\n", package);
3692 printf("**************************************************\n");
3693 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3694 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3695 gSystem->ChangeDirectory(cdir);
3696 return kFALSE;
3697 }
3698 } else {
3699 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3700 gSystem->ChangeDirectory(cdir);
3701 return kFALSE;
3702 }
3703 // Check for SETUP.C and execute
3704 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3705 printf("**************************************************\n");
3706 printf("*** Setup PAR archive %s\n", package);
3707 printf("**************************************************\n");
3708 gROOT->Macro("PROOF-INF/SETUP.C");
3709 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3710 } else {
3711 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3712 gSystem->ChangeDirectory(cdir);
3713 return kFALSE;
3714 }
3715 // Restore original workdir
3716 gSystem->ChangeDirectory(cdir);
3717 return kTRUE;
3718}
3719
c57f56b7 3720//______________________________________________________________________________
3721void AliAnalysisAlien::WriteExecutable()
3722{
3723// Generate the alien executable script.
3724 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3725 ofstream out;
3726 out.open(fExecutable.Data(), ios::out);
3727 if (out.bad()) {
5513444a 3728 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3729 return;
3730 }
3731 out << "#!/bin/bash" << endl;
2a4d5166 3732 // Make sure we can properly compile par files
3733 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3734 out << "echo \"=========================================\"" << endl;
3735 out << "echo \"############## PATH : ##############\"" << endl;
3736 out << "echo $PATH" << endl;
3737 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3738 out << "echo $LD_LIBRARY_PATH" << endl;
3739 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3740 out << "echo $ROOTSYS" << endl;
3741 out << "echo \"############## which root : ##############\"" << endl;
3742 out << "which root" << endl;
3743 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3744 out << "echo $ALICE_ROOT" << endl;
3745 out << "echo \"############## which aliroot : ##############\"" << endl;
3746 out << "which aliroot" << endl;
9c5ddadc 3747 out << "echo \"############## system limits : ##############\"" << endl;
3748 out << "ulimit -a" << endl;
3749 out << "echo \"############## memory : ##############\"" << endl;
3750 out << "free -m" << endl;
c57f56b7 3751 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3752 out << fExecutableCommand << " ";
631c0b05 3753 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3754 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3755 out << "echo \"############## memory after: ##############\"" << endl;
3756 out << "free -m" << endl;
c57f56b7 3757 }
3758 Bool_t copy = kTRUE;
d3b18c4c 3759 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3760 if (copy) {
3761 CdWork();
3762 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3763 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3764 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3765 workdir += fGridWorkingDir;
3766 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3767 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3768 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3769 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3770 }
3771}
3772
0f389141 3773//______________________________________________________________________________
3774void AliAnalysisAlien::WriteMergeExecutable()
3775{
3776// Generate the alien executable script for the merging job.
3777 if (!fMergeViaJDL) return;
3778 TString mergeExec = fExecutable;
3779 mergeExec.ReplaceAll(".sh", "_merge.sh");
3780 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3781 ofstream out;
3782 out.open(mergeExec.Data(), ios::out);
3783 if (out.bad()) {
3784 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3785 return;
3786 }
3787 out << "#!/bin/bash" << endl;
2a4d5166 3788 // Make sure we can properly compile par files
3789 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3790 out << "echo \"=========================================\"" << endl;
3791 out << "echo \"############## PATH : ##############\"" << endl;
3792 out << "echo $PATH" << endl;
3793 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3794 out << "echo $LD_LIBRARY_PATH" << endl;
3795 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3796 out << "echo $ROOTSYS" << endl;
3797 out << "echo \"############## which root : ##############\"" << endl;
3798 out << "which root" << endl;
3799 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3800 out << "echo $ALICE_ROOT" << endl;
3801 out << "echo \"############## which aliroot : ##############\"" << endl;
3802 out << "which aliroot" << endl;
3803 out << "echo \"############## system limits : ##############\"" << endl;
3804 out << "ulimit -a" << endl;
3805 out << "echo \"############## memory : ##############\"" << endl;
3806 out << "free -m" << endl;
3807 out << "echo \"=========================================\"" << endl << endl;
0f389141 3808 TString mergeMacro = fExecutable;
3809 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3810 if (IsOneStageMerging())
3811 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3812 else
f47d5cb4 3813 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3814 out << fExecutableCommand << " " << "$ARG" << endl;
3815 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3816 out << "echo \"############## memory after: ##############\"" << endl;
3817 out << "free -m" << endl;
0f389141 3818 }
3819 Bool_t copy = kTRUE;
d3b18c4c 3820 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3821 if (copy) {
3822 CdWork();
3823 TString workdir = gGrid->GetHomeDirectory();
3824 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3825 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3826 workdir += fGridWorkingDir;
3827 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3828 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3829 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3830 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3831 }
3832}
3833
c57f56b7 3834//______________________________________________________________________________
5513444a 3835void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3836{
3837// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3838// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3839// Next lines: full_path_to_dataset XXX (XXX is a string)
3840// To submit, one has to: submit jdl XXX for all lines
3841 ofstream out;
3842 out.open(filename, ios::out);
3843 if (out.bad()) {
3844 Error("WriteProductionFile", "Bad file name: %s", filename);
3845 return;
3846 }
d3b18c4c 3847 TString workdir;
3848 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3849 workdir = gGrid->GetHomeDirectory();
5513444a 3850 workdir += fGridWorkingDir;
f5e8c702 3851 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3852 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3853 out << locjdl << " " << njobspermaster << endl;
5513444a 3854 Int_t nmasterjobs = fInputFiles->GetEntries();
3855 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3856 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3857 runOutDir.ReplaceAll(".xml", "");
3858 if (fOutputToRunNo)
3859 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3860 else
3861 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3862 }
d3b18c4c 3863 if (gGrid) {
3864 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3865 if (FileExists(filename)) gGrid->Rm(filename);
3866 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3867 }
5513444a 3868}
3869
3870//______________________________________________________________________________
0f389141 3871void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3872{
3873// Generate the alien validation script.
3874 // Generate the validation script
3875 TObjString *os;
d3b18c4c 3876 if (fValidationScript.IsNull()) {
3877 fValidationScript = fExecutable;
3878 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3879 }
3880 TString validationScript = fValidationScript;
3881 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3882 if (!Connect()) {
3883 Error("WriteValidationScript", "Alien connection required");
3884 return;
3885 }
d3b18c4c 3886 if (!fTerminateFiles.IsNull()) {
3887 fTerminateFiles.Strip();
3888 fTerminateFiles.ReplaceAll(" ",",");
3889 }
a2f5fc01 3890 TString outStream = "";
3891 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3892 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3893 ofstream out;
0d5d317c 3894 out.open(validationScript, ios::out);
c57f56b7 3895 out << "#!/bin/bash" << endl;
3896 out << "##################################################" << endl;
3897 out << "validateout=`dirname $0`" << endl;
3898 out << "validatetime=`date`" << endl;
3899 out << "validated=\"0\";" << endl;
3900 out << "error=0" << endl;
3901 out << "if [ -z $validateout ]" << endl;
3902 out << "then" << endl;
3903 out << " validateout=\".\"" << endl;
3904 out << "fi" << endl << endl;
3905 out << "cd $validateout;" << endl;
3906 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3907 out << "echo \"*******************************************************\"" << outStream << endl;
3908 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3909 out << "" << endl;
a2f5fc01 3910 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3911 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3912 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3913 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3914 out << "ls -la ./" << outStream << endl;
3915 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3916 out << "##################################################" << endl;
ebec370a 3917 out << "" << endl;
3918
3919 out << "if [ ! -f stderr ] ; then" << endl;
3920 out << " error=1" << endl;
a2f5fc01 3921 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3922 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3923 out << "fi" << endl;
3924
b34c9f51 3925 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3926 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3927 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3928 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3929 out << "" << endl;
3930
ebec370a 3931 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3932 out << " error=1" << endl;
a2f5fc01 3933 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3934 out << " echo \"$parArch\" " << outStream << endl;
3935 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3936 out << "fi" << endl;
3937
3938 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3939 out << " error=1" << endl;
a2f5fc01 3940 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3941 out << " echo \"$segViol\" " << outStream << endl;
3942 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3943 out << "fi" << endl;
3944
3945 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3946 out << " error=1" << endl;
a2f5fc01 3947 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3948 out << " echo \"$segFault\" " << outStream << endl;
3949 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3950 out << "fi" << endl;
3951
b34c9f51 3952 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3953 out << " error=1" << endl;
a2f5fc01 3954 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3955 out << " echo \"$glibcErr\" " << outStream << endl;
3956 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3957 out << "fi" << endl;
3958
ebec370a 3959 // Part dedicated to the specific analyses running into the train
3960
d3b18c4c 3961 TString outputFiles = fOutputFiles;
3962 if (merge && !fTerminateFiles.IsNull()) {
3963 outputFiles += ",";
3964 outputFiles += fTerminateFiles;
3965 }
3966 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3967 TIter next1(arr);
a2f5fc01 3968 TString outputFile;
bbd88db8 3969 while (!merge && (os=(TObjString*)next1())) {
3970 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3971 outputFile = os->GetString();
3972 Int_t index = outputFile.Index("@");
3973 if (index > 0) outputFile.Remove(index);
bbd88db8 3974 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3975 if (outputFile.Contains("*")) continue;
3976 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3977 out << " error=1" << endl;
7c2cd90a 3978 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3979 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3980 out << "fi" << endl;
3981 }
3982 delete arr;
7c2cd90a 3983 out << "if ! [ -f outputs_valid ] ; then" << endl;
3984 out << " error=1" << endl;
3985 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3986 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3987 out << "fi" << endl;
923e2ca5 3988
c57f56b7 3989 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3990 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3991 if (!IsKeepLogs()) {
3992 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3993 outStream = "";
149d288c 3994 out << " rm -f std*" << endl;
3995 }
c57f56b7 3996 out << "fi" << endl;
3997
a2f5fc01 3998 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3999 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4000 out << "cd -" << endl;
4001 out << "exit $error" << endl;
4002 }
4003 Bool_t copy = kTRUE;
d3b18c4c 4004 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4005 if (copy) {
4006 CdWork();
4007 TString workdir = gGrid->GetHomeDirectory();
4008 workdir += fGridWorkingDir;
d3b18c4c 4009 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4010 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4011 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4012 }
4013}