]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Loading libOADB in the generated macros of the plugin
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
7c2cd90a 27#include "TBits.h"
0f389141 28#include "TError.h"
c57f56b7 29#include "TROOT.h"
30#include "TSystem.h"
31#include "TFile.h"
830acc4c 32#include "TFileCollection.h"
3bdcb562 33#include "TChain.h"
c57f56b7 34#include "TObjString.h"
35#include "TObjArray.h"
36#include "TGrid.h"
37#include "TGridResult.h"
38#include "TGridCollection.h"
39#include "TGridJDL.h"
d2a409b2 40#include "TGridJobStatusList.h"
41#include "TGridJobStatus.h"
c57f56b7 42#include "TFileMerger.h"
43#include "AliAnalysisManager.h"
bb885a9e 44#include "AliVEventHandler.h"
45#include "AliAnalysisDataContainer.h"
65eb22f8 46#include "AliMultiInputEventHandler.h"
c57f56b7 47
48ClassImp(AliAnalysisAlien)
49
50//______________________________________________________________________________
51AliAnalysisAlien::AliAnalysisAlien()
52 :AliAnalysisGrid(),
53 fGridJDL(NULL),
0f389141 54 fMergingJDL(NULL),
c57f56b7 55 fPrice(0),
56 fTTL(0),
57 fSplitMaxInputFileNumber(0),
58 fMaxInitFailed(0),
59 fMasterResubmitThreshold(0),
bb885a9e 60 fNtestFiles(0),
319593fb 61 fNrunsPerMaster(0),
16a4353c 62 fMaxMergeFiles(0),
b3e07543 63 fMaxMergeStages(0),
d2a409b2 64 fNsubmitted(0),
a3e84053 65 fProductionMode(0),
cd11251e 66 fOutputToRunNo(0),
0f389141 67 fMergeViaJDL(0),
68 fFastReadOption(0),
e1c22e21 69 fOverwriteMode(1),
149d288c 70 fNreplicas(2),
3bdcb562 71 fNproofWorkers(0),
27734f0e 72 fNproofWorkersPerSlave(0),
3bdcb562 73 fProofReset(0),
c57f56b7 74 fRunNumbers(),
75 fExecutable(),
0a1c1f7f 76 fExecutableCommand(),
c57f56b7 77 fArguments(),
631c0b05 78 fExecutableArgs(),
c57f56b7 79 fAnalysisMacro(),
80 fAnalysisSource(),
d3b18c4c 81 fValidationScript(),
d5c6455a 82 fAdditionalRootLibs(),
c57f56b7 83 fAdditionalLibs(),
84 fSplitMode(),
85 fAPIVersion(),
86 fROOTVersion(),
87 fAliROOTVersion(),
648174cf 88 fExternalPackages(),
c57f56b7 89 fUser(),
90 fGridWorkingDir(),
91 fGridDataDir(),
92 fDataPattern(),
93 fGridOutputDir(),
94 fOutputArchive(),
95 fOutputFiles(),
96 fInputFormat(),
e7c71df0 97 fDatasetName(),
c57f56b7 98 fJDLName(),
d3b18c4c 99 fTerminateFiles(),
bb885a9e 100 fMergeExcludes(),
f965131e 101 fIncludePath(),
bb885a9e 102 fCloseSE(),
0df6ccf2 103 fFriendChainName(),
c6cb3634 104 fJobTag(),
648174cf 105 fOutputSingle(),
5fce53f4 106 fRunPrefix(),
3bdcb562 107 fProofCluster(),
108 fProofDataSet(),
109 fFileForTestMode(),
110 fRootVersionForProof(),
111 fAliRootMode(),
f47d5cb4 112 fMergeDirName(),
4e5c5506 113 fInputFiles(0),
114 fPackages(0)
c57f56b7 115{
116// Dummy ctor.
117 SetDefaults();
118}
119
120//______________________________________________________________________________
121AliAnalysisAlien::AliAnalysisAlien(const char *name)
122 :AliAnalysisGrid(name),
123 fGridJDL(NULL),
0f389141 124 fMergingJDL(NULL),
c57f56b7 125 fPrice(0),
126 fTTL(0),
127 fSplitMaxInputFileNumber(0),
128 fMaxInitFailed(0),
129 fMasterResubmitThreshold(0),
bb885a9e 130 fNtestFiles(0),
319593fb 131 fNrunsPerMaster(0),
16a4353c 132 fMaxMergeFiles(0),
b3e07543 133 fMaxMergeStages(0),
d2a409b2 134 fNsubmitted(0),
a3e84053 135 fProductionMode(0),
cd11251e 136 fOutputToRunNo(0),
0f389141 137 fMergeViaJDL(0),
138 fFastReadOption(0),
e1c22e21 139 fOverwriteMode(1),
149d288c 140 fNreplicas(2),
3bdcb562 141 fNproofWorkers(0),
27734f0e 142 fNproofWorkersPerSlave(0),
3bdcb562 143 fProofReset(0),
c57f56b7 144 fRunNumbers(),
145 fExecutable(),
0a1c1f7f 146 fExecutableCommand(),
c57f56b7 147 fArguments(),
631c0b05 148 fExecutableArgs(),
c57f56b7 149 fAnalysisMacro(),
150 fAnalysisSource(),
d3b18c4c 151 fValidationScript(),
d5c6455a 152 fAdditionalRootLibs(),
c57f56b7 153 fAdditionalLibs(),
154 fSplitMode(),
155 fAPIVersion(),
156 fROOTVersion(),
157 fAliROOTVersion(),
648174cf 158 fExternalPackages(),
c57f56b7 159 fUser(),
160 fGridWorkingDir(),
161 fGridDataDir(),
162 fDataPattern(),
163 fGridOutputDir(),
164 fOutputArchive(),
165 fOutputFiles(),
166 fInputFormat(),
e7c71df0 167 fDatasetName(),
c57f56b7 168 fJDLName(),
d3b18c4c 169 fTerminateFiles(),
bb885a9e 170 fMergeExcludes(),
f965131e 171 fIncludePath(),
bb885a9e 172 fCloseSE(),
0df6ccf2 173 fFriendChainName(),
c6cb3634 174 fJobTag(),
648174cf 175 fOutputSingle(),
5fce53f4 176 fRunPrefix(),
3bdcb562 177 fProofCluster(),
178 fProofDataSet(),
179 fFileForTestMode(),
180 fRootVersionForProof(),
181 fAliRootMode(),
f47d5cb4 182 fMergeDirName(),
4e5c5506 183 fInputFiles(0),
184 fPackages(0)
c57f56b7 185{
186// Default ctor.
187 SetDefaults();
188}
189
190//______________________________________________________________________________
191AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
192 :AliAnalysisGrid(other),
193 fGridJDL(NULL),
0f389141 194 fMergingJDL(NULL),
c57f56b7 195 fPrice(other.fPrice),
196 fTTL(other.fTTL),
197 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
198 fMaxInitFailed(other.fMaxInitFailed),
199 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 200 fNtestFiles(other.fNtestFiles),
319593fb 201 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 202 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 203 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 204 fNsubmitted(other.fNsubmitted),
a3e84053 205 fProductionMode(other.fProductionMode),
cd11251e 206 fOutputToRunNo(other.fOutputToRunNo),
0f389141 207 fMergeViaJDL(other.fMergeViaJDL),
208 fFastReadOption(other.fFastReadOption),
209 fOverwriteMode(other.fOverwriteMode),
149d288c 210 fNreplicas(other.fNreplicas),
3bdcb562 211 fNproofWorkers(other.fNproofWorkers),
27734f0e 212 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 213 fProofReset(other.fProofReset),
c57f56b7 214 fRunNumbers(other.fRunNumbers),
215 fExecutable(other.fExecutable),
0a1c1f7f 216 fExecutableCommand(other.fExecutableCommand),
c57f56b7 217 fArguments(other.fArguments),
631c0b05 218 fExecutableArgs(other.fExecutableArgs),
c57f56b7 219 fAnalysisMacro(other.fAnalysisMacro),
220 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 221 fValidationScript(other.fValidationScript),
d5c6455a 222 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 223 fAdditionalLibs(other.fAdditionalLibs),
224 fSplitMode(other.fSplitMode),
225 fAPIVersion(other.fAPIVersion),
226 fROOTVersion(other.fROOTVersion),
227 fAliROOTVersion(other.fAliROOTVersion),
648174cf 228 fExternalPackages(other.fExternalPackages),
c57f56b7 229 fUser(other.fUser),
230 fGridWorkingDir(other.fGridWorkingDir),
231 fGridDataDir(other.fGridDataDir),
232 fDataPattern(other.fDataPattern),
233 fGridOutputDir(other.fGridOutputDir),
234 fOutputArchive(other.fOutputArchive),
235 fOutputFiles(other.fOutputFiles),
236 fInputFormat(other.fInputFormat),
e7c71df0 237 fDatasetName(other.fDatasetName),
c57f56b7 238 fJDLName(other.fJDLName),
d3b18c4c 239 fTerminateFiles(other.fTerminateFiles),
bb885a9e 240 fMergeExcludes(other.fMergeExcludes),
f965131e 241 fIncludePath(other.fIncludePath),
bb885a9e 242 fCloseSE(other.fCloseSE),
0df6ccf2 243 fFriendChainName(other.fFriendChainName),
c6cb3634 244 fJobTag(other.fJobTag),
648174cf 245 fOutputSingle(other.fOutputSingle),
5fce53f4 246 fRunPrefix(other.fRunPrefix),
3bdcb562 247 fProofCluster(other.fProofCluster),
248 fProofDataSet(other.fProofDataSet),
249 fFileForTestMode(other.fFileForTestMode),
250 fRootVersionForProof(other.fRootVersionForProof),
251 fAliRootMode(other.fAliRootMode),
f47d5cb4 252 fMergeDirName(other.fMergeDirName),
4e5c5506 253 fInputFiles(0),
254 fPackages(0)
c57f56b7 255{
256// Copy ctor.
257 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 258 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 259 fRunRange[0] = other.fRunRange[0];
260 fRunRange[1] = other.fRunRange[1];
c57f56b7 261 if (other.fInputFiles) {
262 fInputFiles = new TObjArray();
263 TIter next(other.fInputFiles);
264 TObject *obj;
265 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
266 fInputFiles->SetOwner();
267 }
4e5c5506 268 if (other.fPackages) {
269 fPackages = new TObjArray();
270 TIter next(other.fPackages);
271 TObject *obj;
272 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
273 fPackages->SetOwner();
274 }
c57f56b7 275}
276
277//______________________________________________________________________________
278AliAnalysisAlien::~AliAnalysisAlien()
279{
280// Destructor.
281 if (fGridJDL) delete fGridJDL;
0f389141 282 if (fMergingJDL) delete fMergingJDL;
c57f56b7 283 if (fInputFiles) delete fInputFiles;
4e5c5506 284 if (fPackages) delete fPackages;
c57f56b7 285}
286
287//______________________________________________________________________________
288AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
289{
290// Assignment.
291 if (this != &other) {
292 AliAnalysisGrid::operator=(other);
293 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 294 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 295 fPrice = other.fPrice;
296 fTTL = other.fTTL;
297 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
298 fMaxInitFailed = other.fMaxInitFailed;
299 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 300 fNtestFiles = other.fNtestFiles;
a3e84053 301 fNrunsPerMaster = other.fNrunsPerMaster;
302 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 303 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 304 fNsubmitted = other.fNsubmitted;
305 fProductionMode = other.fProductionMode;
cd11251e 306 fOutputToRunNo = other.fOutputToRunNo;
0f389141 307 fMergeViaJDL = other.fMergeViaJDL;
308 fFastReadOption = other.fFastReadOption;
309 fOverwriteMode = other.fOverwriteMode;
149d288c 310 fNreplicas = other.fNreplicas;
3bdcb562 311 fNproofWorkers = other.fNproofWorkers;
27734f0e 312 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 313 fProofReset = other.fProofReset;
c57f56b7 314 fRunNumbers = other.fRunNumbers;
315 fExecutable = other.fExecutable;
0a1c1f7f 316 fExecutableCommand = other.fExecutableCommand;
c57f56b7 317 fArguments = other.fArguments;
631c0b05 318 fExecutableArgs = other.fExecutableArgs;
c57f56b7 319 fAnalysisMacro = other.fAnalysisMacro;
320 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 321 fValidationScript = other.fValidationScript;
d5c6455a 322 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 323 fAdditionalLibs = other.fAdditionalLibs;
324 fSplitMode = other.fSplitMode;
325 fAPIVersion = other.fAPIVersion;
326 fROOTVersion = other.fROOTVersion;
327 fAliROOTVersion = other.fAliROOTVersion;
648174cf 328 fExternalPackages = other.fExternalPackages;
c57f56b7 329 fUser = other.fUser;
330 fGridWorkingDir = other.fGridWorkingDir;
331 fGridDataDir = other.fGridDataDir;
332 fDataPattern = other.fDataPattern;
333 fGridOutputDir = other.fGridOutputDir;
334 fOutputArchive = other.fOutputArchive;
335 fOutputFiles = other.fOutputFiles;
336 fInputFormat = other.fInputFormat;
e7c71df0 337 fDatasetName = other.fDatasetName;
c57f56b7 338 fJDLName = other.fJDLName;
d3b18c4c 339 fTerminateFiles = other.fTerminateFiles;
bb885a9e 340 fMergeExcludes = other.fMergeExcludes;
f965131e 341 fIncludePath = other.fIncludePath;
bb885a9e 342 fCloseSE = other.fCloseSE;
0df6ccf2 343 fFriendChainName = other.fFriendChainName;
c6cb3634 344 fJobTag = other.fJobTag;
648174cf 345 fOutputSingle = other.fOutputSingle;
5fce53f4 346 fRunPrefix = other.fRunPrefix;
3bdcb562 347 fProofCluster = other.fProofCluster;
348 fProofDataSet = other.fProofDataSet;
349 fFileForTestMode = other.fFileForTestMode;
350 fRootVersionForProof = other.fRootVersionForProof;
351 fAliRootMode = other.fAliRootMode;
f47d5cb4 352 fMergeDirName = other.fMergeDirName;
c57f56b7 353 if (other.fInputFiles) {
354 fInputFiles = new TObjArray();
355 TIter next(other.fInputFiles);
356 TObject *obj;
357 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
358 fInputFiles->SetOwner();
359 }
4e5c5506 360 if (other.fPackages) {
361 fPackages = new TObjArray();
362 TIter next(other.fPackages);
363 TObject *obj;
364 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
365 fPackages->SetOwner();
366 }
c57f56b7 367 }
368 return *this;
369}
370
74013535 371//______________________________________________________________________________
372void AliAnalysisAlien::SetRunPrefix(const char *prefix)
373{
374// Set the run number format. Can be a prefix or a format like "%09d"
375 fRunPrefix = prefix;
376 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
377}
378
f965131e 379//______________________________________________________________________________
380void AliAnalysisAlien::AddIncludePath(const char *path)
381{
382// Add include path in the remote analysis macro.
383 TString p(path);
384 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
385 else fIncludePath += Form("-I%s ", path);
386}
387
c57f56b7 388//______________________________________________________________________________
389void AliAnalysisAlien::AddRunNumber(Int_t run)
390{
391// Add a run number to the list of runs to be processed.
392 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 393 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 394}
395
98690895 396//______________________________________________________________________________
397void AliAnalysisAlien::AddRunList(const char* runList)
398{
399// Add several runs into the list of runs; they are expected to be separated by a blank character.
400 TString sList = runList;
401 TObjArray *list = sList.Tokenize(" ");
402 Int_t n = list->GetEntries();
403 for (Int_t i = 0; i < n; i++) {
404 TObjString *os = (TObjString*)list->At(i);
405 AddRunNumber(os->GetString().Atoi());
406 }
407 delete list;
408}
409
ee75cfc3 410//______________________________________________________________________________
411void AliAnalysisAlien::AddRunNumber(const char* run)
412{
413// Add a run number to the list of runs to be processed.
7aa5d539 414 TString runs = run;
415 TObjString *os;
416 TObjArray *arr = runs.Tokenize(" ");
417 TIter next(arr);
418 TString prefix;
419 prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
420 while ((os=(TObjString*)next())){
421 if (fRunNumbers.Length()) fRunNumbers += " ";
422 fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
423 }
424 delete arr;
ee75cfc3 425}
426
c57f56b7 427//______________________________________________________________________________
428void AliAnalysisAlien::AddDataFile(const char *lfn)
429{
430// Adds a data file to the input to be analysed. The file should be a valid LFN
431// or point to an existing file in the alien workdir.
432 if (!fInputFiles) fInputFiles = new TObjArray();
433 fInputFiles->Add(new TObjString(lfn));
434}
648174cf 435
436//______________________________________________________________________________
437void AliAnalysisAlien::AddExternalPackage(const char *package)
438{
439// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
440 if (fExternalPackages) fExternalPackages += " ";
441 fExternalPackages += package;
442}
443
c57f56b7 444//______________________________________________________________________________
445Bool_t AliAnalysisAlien::Connect()
446{
447// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
448 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 449 if (fProductionMode) return kTRUE;
c57f56b7 450 if (!gGrid) {
451 Info("Connect", "Trying to connect to AliEn ...");
452 TGrid::Connect("alien://");
453 }
454 if (!gGrid || !gGrid->IsConnected()) {
455 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
456 return kFALSE;
457 }
458 fUser = gGrid->GetUser();
459 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
460 return kTRUE;
461}
462
463//______________________________________________________________________________
464void AliAnalysisAlien::CdWork()
465{
466// Check validity of alien workspace. Create directory if possible.
467 if (!Connect()) {
468 Error("CdWork", "Alien connection required");
469 return;
470 }
471 TString homedir = gGrid->GetHomeDirectory();
472 TString workdir = homedir + fGridWorkingDir;
923e2ca5 473 if (DirectoryExists(workdir)) {
474 gGrid->Cd(workdir);
475 return;
476 }
477 // Work directory not existing - create it
478 gGrid->Cd(homedir);
b93f8109 479 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 480 gGrid->Cd(fGridWorkingDir);
d3b18c4c 481 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 482 } else {
d3b18c4c 483 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 484 workdir.Data(), homedir.Data());
485 fGridWorkingDir = "";
486 }
c57f56b7 487}
488
348be253 489//______________________________________________________________________________
490Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
491{
492// Check if file copying is possible.
d3b18c4c 493 if (fProductionMode) return kTRUE;
348be253 494 if (!Connect()) {
495 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
496 return kFALSE;
497 }
d3339be3 498 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
499 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 500 // Check if alien_CLOSE_SE is defined
501 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
502 if (!closeSE.IsNull()) {
503 Info("CheckFileCopy", "Your current close storage is pointing to: \
504 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
505 } else {
506 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
507 }
508 // Check if grid directory exists.
509 if (!DirectoryExists(alienpath)) {
510 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
511 return kFALSE;
512 }
d3339be3 513 TFile f("plugin_test_copy", "RECREATE");
348be253 514 // User may not have write permissions to current directory
515 if (f.IsZombie()) {
516 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
517 gSystem->WorkingDirectory());
518 return kFALSE;
519 }
520 f.Close();
d3339be3 521 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
522 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
523 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 524 \n# 1. Make sure you have write permissions there. If this is the case: \
525 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
526 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
527 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
528 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
529 gSystem->Unlink(f.GetName());
530 return kFALSE;
531 }
532 gSystem->Unlink(f.GetName());
d3339be3 533 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
534 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 535 return kTRUE;
536}
537
c57f56b7 538//______________________________________________________________________________
539Bool_t AliAnalysisAlien::CheckInputData()
540{
541// Check validity of input data. If necessary, create xml files.
d3b18c4c 542 if (fProductionMode) return kTRUE;
a8739e8a 543 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 544 if (!fGridDataDir.Length()) {
545 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
546 return kFALSE;
547 }
9f020952 548 if (fMergeViaJDL) {
549 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
550 return kFALSE;
551 }
d2a409b2 552 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 553 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
554 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 555 return kTRUE;
c57f56b7 556 }
557 // Process declared files
a2f5fc01 558 Bool_t isCollection = kFALSE;
559 Bool_t isXml = kFALSE;
560 Bool_t useTags = kFALSE;
c57f56b7 561 Bool_t checked = kFALSE;
d3b18c4c 562 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 563 TString file;
564 TString workdir = gGrid->GetHomeDirectory();
565 workdir += fGridWorkingDir;
566 if (fInputFiles) {
567 TObjString *objstr;
568 TIter next(fInputFiles);
569 while ((objstr=(TObjString*)next())) {
570 file = workdir;
571 file += "/";
572 file += objstr->GetString();
573 // Store full lfn path
574 if (FileExists(file)) objstr->SetString(file);
575 else {
576 file = objstr->GetName();
577 if (!FileExists(objstr->GetName())) {
578 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
579 objstr->GetName(), workdir.Data());
580 return kFALSE;
581 }
582 }
583 Bool_t iscoll, isxml, usetags;
584 CheckDataType(file, iscoll, isxml, usetags);
585 if (!checked) {
586 checked = kTRUE;
a2f5fc01 587 isCollection = iscoll;
588 isXml = isxml;
589 useTags = usetags;
590 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 591 } else {
a2f5fc01 592 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 593 Error("CheckInputData", "Some conflict was found in the types of inputs");
594 return kFALSE;
595 }
596 }
597 }
598 }
599 // Process requested run numbers
a8739e8a 600 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 601 // Check validity of alien data directory
602 if (!fGridDataDir.Length()) {
603 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
604 return kFALSE;
605 }
923e2ca5 606 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 607 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
608 return kFALSE;
609 }
a2f5fc01 610 if (isCollection) {
c57f56b7 611 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
612 return kFALSE;
613 }
614
a2f5fc01 615 if (checked && !isXml) {
c57f56b7 616 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
617 return kFALSE;
618 }
619 // Check validity of run number(s)
620 TObjArray *arr;
621 TObjString *os;
74013535 622 TString format;
319593fb 623 Int_t nruns = 0;
904f9f5f 624 TString schunk, schunk2;
c57f56b7 625 TString path;
626 if (!checked) {
627 checked = kTRUE;
a2f5fc01 628 useTags = fDataPattern.Contains("tag");
629 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 630 }
a2f5fc01 631 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 632 Error("CheckInputData", "Cannot mix input files using/not using tags");
633 return kFALSE;
634 }
635 if (fRunNumbers.Length()) {
a8739e8a 636 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 637 arr = fRunNumbers.Tokenize(" ");
638 TIter next(arr);
639 while ((os=(TObjString*)next())) {
640 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 641 if (!DirectoryExists(path)) {
642 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 643 continue;
c57f56b7 644 }
645 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
646 TString msg = "\n##### file: ";
647 msg += path;
648 msg += " type: xml_collection;";
a2f5fc01 649 if (useTags) msg += " using_tags: Yes";
c57f56b7 650 else msg += " using_tags: No";
23329835 651 Info("CheckDataType", "%s", msg.Data());
319593fb 652 if (fNrunsPerMaster<2) {
d2a409b2 653 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 654 } else {
655 nruns++;
656 if (((nruns-1)%fNrunsPerMaster) == 0) {
657 schunk = os->GetString();
658 }
659 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
660 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 661 AddDataFile(schunk);
319593fb 662 }
c57f56b7 663 }
664 delete arr;
a8739e8a 665 } else {
666 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
667 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 668 format = Form("%%s/%s ", fRunPrefix.Data());
669 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 670 if (!DirectoryExists(path)) {
a8739e8a 671 continue;
672 }
74013535 673 format = Form("%%s/%s.xml", fRunPrefix.Data());
674 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 675 TString msg = "\n##### file: ";
676 msg += path;
677 msg += " type: xml_collection;";
a2f5fc01 678 if (useTags) msg += " using_tags: Yes";
a8739e8a 679 else msg += " using_tags: No";
23329835 680 Info("CheckDataType", "%s", msg.Data());
319593fb 681 if (fNrunsPerMaster<2) {
74013535 682 format = Form("%s.xml", fRunPrefix.Data());
683 AddDataFile(Form(format.Data(),irun));
319593fb 684 } else {
685 nruns++;
686 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 687 schunk = Form(fRunPrefix.Data(),irun);
319593fb 688 }
74013535 689 format = Form("_%s.xml", fRunPrefix.Data());
690 schunk2 = Form(format.Data(), irun);
319593fb 691 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 692 schunk += schunk2;
d2a409b2 693 AddDataFile(schunk);
319593fb 694 }
a8739e8a 695 }
904f9f5f 696 if (!fInputFiles) {
697 schunk += schunk2;
698 AddDataFile(schunk);
699 }
c57f56b7 700 }
701 return kTRUE;
702}
703
704//______________________________________________________________________________
705Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
706{
707// Create dataset for the grid data directory + run number.
d3b18c4c 708 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 709 if (!Connect()) {
710 Error("CreateDataset", "Cannot create dataset with no grid connection");
711 return kFALSE;
712 }
713
714 // Cd workspace
d3b18c4c 715 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 716 TString workdir = gGrid->GetHomeDirectory();
717 workdir += fGridWorkingDir;
718
719 // Compose the 'find' command arguments
74013535 720 TString format;
c57f56b7 721 TString command;
722 TString options = "-x collection ";
bb885a9e 723 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 724 TString conditions = "";
725
726 TString file;
727 TString path;
319593fb 728 Int_t nruns = 0;
904f9f5f 729 TString schunk, schunk2;
ab254fd1 730 TGridCollection *cbase=0, *cadd=0;
d2a409b2 731 if (!fRunNumbers.Length() && !fRunRange[0]) {
732 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
733 // Make a single data collection from data directory.
734 path = fGridDataDir;
923e2ca5 735 if (!DirectoryExists(path)) {
d2a409b2 736 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
737 return kFALSE;
738 }
923e2ca5 739// CdWork();
d2a409b2 740 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
741 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 742 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 743 command = "find ";
744 command += options;
745 command += path;
746 command += " ";
747 command += pattern;
748 command += conditions;
84fcd93f 749 printf("command: %s\n", command.Data());
d2a409b2 750 TGridResult *res = gGrid->Command(command);
751 if (res) delete res;
752 // Write standard output to file
753 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 754 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
755 Bool_t nullFile = kFALSE;
756 if (!hasGrep) {
d3339be3 757 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
758 } else {
a2f5fc01 759 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
760 if (nullFile) {
defd7a3a 761 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
762 return kFALSE;
763 }
d3339be3 764 }
0f389141 765 }
766 Bool_t fileExists = FileExists(file);
767 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 768 // Copy xml file to alien space
0f389141 769 if (fileExists) gGrid->Rm(file);
d2a409b2 770 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
771 if (!FileExists(file)) {
772 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
773 return kFALSE;
774 }
775 // Update list of files to be processed.
776 }
777 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
778 return kTRUE;
779 }
c57f56b7 780 // Several runs
a2f5fc01 781 Bool_t nullResult = kTRUE;
a8739e8a 782 if (fRunNumbers.Length()) {
783 TObjArray *arr = fRunNumbers.Tokenize(" ");
784 TObjString *os;
785 TIter next(arr);
786 while ((os=(TObjString*)next())) {
dc030a24 787 path = Form("%s/%s/ ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 788 if (!DirectoryExists(path)) continue;
789// CdWork();
a8739e8a 790 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
791 else file = Form("%s.xml", os->GetString().Data());
319593fb 792 // If local collection file does not exist, create it via 'find' command.
d3339be3 793 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 794 command = "find ";
795 command += options;
796 command += path;
797 command += pattern;
798 command += conditions;
799 TGridResult *res = gGrid->Command(command);
800 if (res) delete res;
801 // Write standard output to file
802 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 803 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
804 Bool_t nullFile = kFALSE;
805 if (!hasGrep) {
d3339be3 806 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
807 } else {
a2f5fc01 808 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
809 if (nullFile) {
defd7a3a 810 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
811 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
812 continue;
813 }
d3339be3 814 }
a2f5fc01 815 nullResult = kFALSE;
d3339be3 816 }
a8739e8a 817 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 818 // Check if there is one run per master job.
819 if (fNrunsPerMaster<2) {
820 if (FileExists(file)) {
0f389141 821 if (fOverwriteMode) gGrid->Rm(file);
822 else {
823 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
824 continue;
825 }
319593fb 826 }
827 // Copy xml file to alien space
828 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
829 if (!FileExists(file)) {
830 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
831 delete arr;
832 return kFALSE;
833 }
834 } else {
835 nruns++;
836 if (((nruns-1)%fNrunsPerMaster) == 0) {
837 schunk = os->GetString();
838 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
839 } else {
840 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 841 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 842 cbase->Add(cadd);
843 delete cadd;
844 }
845 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
846 continue;
847 }
848 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 849 if (FileExists(schunk)) {
850 if (fOverwriteMode) gGrid->Rm(file);
851 else {
852 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
853 continue;
854 }
319593fb 855 }
84fcd93f 856 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 857 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 858 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 859 if (!FileExists(schunk)) {
860 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
861 delete arr;
862 return kFALSE;
863 }
d3339be3 864 }
a8739e8a 865 }
866 delete arr;
a2f5fc01 867 if (nullResult) {
d3339be3 868 Error("CreateDataset", "No valid dataset corresponding to the query!");
869 return kFALSE;
870 }
a8739e8a 871 } else {
872 // Process a full run range.
873 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 874 format = Form("%%s/%s ", fRunPrefix.Data());
875 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 876 if (!DirectoryExists(path)) continue;
877// CdWork();
74013535 878 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 879 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 880 else file = Form(format.Data(), irun);
0f389141 881 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
882 if (fOverwriteMode) gGrid->Rm(file);
883 else {
884 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
885 continue;
886 }
a8739e8a 887 }
319593fb 888 // If local collection file does not exist, create it via 'find' command.
d3339be3 889 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 890 command = "find ";
891 command += options;
892 command += path;
893 command += pattern;
894 command += conditions;
895 TGridResult *res = gGrid->Command(command);
896 if (res) delete res;
897 // Write standard output to file
898 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 899 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
900 Bool_t nullFile = kFALSE;
901 if (!hasGrep) {
d3339be3 902 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
903 } else {
a2f5fc01 904 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
905 if (nullFile) {
defd7a3a 906 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
907 continue;
908 }
d3339be3 909 }
a2f5fc01 910 nullResult = kFALSE;
319593fb 911 }
a8739e8a 912 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 913 // Check if there is one run per master job.
914 if (fNrunsPerMaster<2) {
915 if (FileExists(file)) {
0f389141 916 if (fOverwriteMode) gGrid->Rm(file);
917 else {
918 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
919 continue;
920 }
319593fb 921 }
922 // Copy xml file to alien space
923 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
924 if (!FileExists(file)) {
925 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
926 return kFALSE;
927 }
928 } else {
929 nruns++;
95e5b448 930 // Check if the collection for the chunk exist locally.
931 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 932 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
933 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
934 else continue;
935 }
84fcd93f 936 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 937 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 938 schunk = Form(fRunPrefix.Data(), irun);
319593fb 939 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
940 } else {
941 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
942 cbase->Add(cadd);
943 delete cadd;
944 }
74013535 945 format = Form("%%s_%s.xml", fRunPrefix.Data());
946 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 947 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 948 continue;
949 }
904f9f5f 950 schunk = schunk2;
319593fb 951 if (FileExists(schunk)) {
0f389141 952 if (fOverwriteMode) gGrid->Rm(schunk);
953 else {
954 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
955 continue;
956 }
319593fb 957 }
84fcd93f 958 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 959 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 960 if (FileExists(schunk)) {
0f389141 961 if (fOverwriteMode) gGrid->Rm(schunk);
962 else {
963 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
964 continue;
965 }
95e5b448 966 }
319593fb 967 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
968 if (!FileExists(schunk)) {
969 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
970 return kFALSE;
971 }
972 }
c57f56b7 973 }
a2f5fc01 974 if (nullResult) {
d3339be3 975 Error("CreateDataset", "No valid dataset corresponding to the query!");
976 return kFALSE;
977 }
a8739e8a 978 }
c57f56b7 979 return kTRUE;
980}
981
982//______________________________________________________________________________
983Bool_t AliAnalysisAlien::CreateJDL()
984{
985// Generate a JDL file according to current settings. The name of the file is
986// specified by fJDLName.
987 Bool_t error = kFALSE;
988 TObjArray *arr = 0;
989 Bool_t copy = kTRUE;
d3b18c4c 990 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 991 Bool_t generate = kTRUE;
992 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
993 if (!Connect()) {
994 Error("CreateJDL", "Alien connection required");
995 return kFALSE;
996 }
997 // Check validity of alien workspace
d3b18c4c 998 TString workdir;
999 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1000 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 1001 workdir += fGridWorkingDir;
1002 if (generate) {
1003 TObjString *os;
1004 if (!fInputFiles) {
1005 Error("CreateJDL()", "Define some input files for your analysis.");
1006 error = kTRUE;
1007 }
1008 // Compose list of input files
1009 // Check if output files were defined
1010 if (!fOutputFiles.Length()) {
1011 Error("CreateJDL", "You must define at least one output file");
1012 error = kTRUE;
1013 }
1014 // Check if an output directory was defined and valid
1015 if (!fGridOutputDir.Length()) {
1016 Error("CreateJDL", "You must define AliEn output directory");
1017 error = kTRUE;
1018 } else {
d3b18c4c 1019 if (!fProductionMode) {
1020 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1021 if (!DirectoryExists(fGridOutputDir)) {
1022 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1023 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1024 } else {
1025 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1026 // error = kTRUE;
1027 }
67a3b254 1028 } else {
1029 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1030 }
1031 gGrid->Cd(workdir);
1032 }
c57f56b7 1033 }
1034 // Exit if any error up to now
1035 if (error) return kFALSE;
1036 // Set JDL fields
0f389141 1037 if (!fUser.IsNull()) {
1038 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1039 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1040 }
1041 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1042 TString mergeExec = fExecutable;
1043 mergeExec.ReplaceAll(".sh", "_merge.sh");
1044 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1045 mergeExec.ReplaceAll(".sh", ".C");
1046 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1047 if (!fArguments.IsNull())
1048 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1049 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1050 else {
1051 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1052 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1053 }
b3e07543 1054
1f0d1ca2 1055 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1056 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1057 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1058 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1059
0f389141 1060 if (fMaxInitFailed > 0) {
c57f56b7 1061 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1062 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1063 }
1064 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1065 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1066 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1067 }
1068 if (!IsOneStageMerging()) {
f47d5cb4 1069 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1070 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1071 }
1072 if (fSplitMode.Length()) {
c57f56b7 1073 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1074 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1075 }
1076 fMergingJDL->SetValue("Split", "\"se\"");
1077 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1078 if (!fAliROOTVersion.IsNull()) {
1079 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1080 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1081 }
1082 if (!fROOTVersion.IsNull()) {
c57f56b7 1083 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1084 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1085 }
1086 if (!fAPIVersion.IsNull()) {
c57f56b7 1087 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1088 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1089 }
648174cf 1090 if (!fExternalPackages.IsNull()) {
1091 arr = fExternalPackages.Tokenize(" ");
1092 TIter next(arr);
1093 while ((os=(TObjString*)next())) {
1094 TString pkgname = os->GetString();
1095 Int_t index = pkgname.Index("::");
1096 TString pkgversion = pkgname(index+2, pkgname.Length());
1097 pkgname.Remove(index);
1098 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1099 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1100 }
1101 delete arr;
1102 }
0f389141 1103 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1104 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1105 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1106 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1107 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1108 TString analysisFile = fExecutable;
1109 analysisFile.ReplaceAll(".sh", ".root");
1110 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1111 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1112 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1113 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1114 if (fAdditionalLibs.Length()) {
1115 arr = fAdditionalLibs.Tokenize(" ");
1116 TIter next(arr);
1117 while ((os=(TObjString*)next())) {
1118 if (os->GetString().Contains(".so")) continue;
1119 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1120 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1121 }
1122 delete arr;
1123 }
4e5c5506 1124 if (fPackages) {
1125 TIter next(fPackages);
1126 TObject *obj;
0f389141 1127 while ((obj=next())) {
4e5c5506 1128 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1129 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1130 }
4e5c5506 1131 }
c57f56b7 1132 if (fOutputArchive.Length()) {
1133 arr = fOutputArchive.Tokenize(" ");
1134 TIter next(arr);
0f389141 1135 Bool_t first = kTRUE;
1136 const char *comment = "Files to be archived";
1137 const char *comment1 = comment;
1138 while ((os=(TObjString*)next())) {
1139 if (!first) comment = NULL;
1140 if (!os->GetString().Contains("@") && fCloseSE.Length())
1141 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1142 else
1143 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1144 first = kFALSE;
1145 }
c57f56b7 1146 delete arr;
f790bc1b 1147 // Output archive for the merging jdl
1148 TString outputArchive;
1149 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1150 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1151 // Add normal output files, extra files + terminate files
1152 TString files = GetListOfFiles("outextter");
1153 // Do not register merge excludes
1154 if (!fMergeExcludes.IsNull()) {
1155 arr = fMergeExcludes.Tokenize(" ");
1156 TIter next1(arr);
1157 while ((os=(TObjString*)next1())) {
1158 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1159 files.ReplaceAll(os->GetString(),"");
1160 }
66d71516 1161 delete arr;
0f389141 1162 }
f790bc1b 1163 files.ReplaceAll(".root", "*.root");
b83f84f0 1164 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1165 } else {
95f84779 1166 TString files = fOutputArchive;
1167 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1168 outputArchive = files;
f790bc1b 1169 }
0f389141 1170 arr = outputArchive.Tokenize(" ");
705adb3e 1171 TIter next2(arr);
0f389141 1172 comment = comment1;
1173 first = kTRUE;
705adb3e 1174 while ((os=(TObjString*)next2())) {
0f389141 1175 if (!first) comment = NULL;
0cdf65a8 1176 TString currentfile = os->GetString();
0cdf65a8 1177 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1178 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1179 else
0cdf65a8 1180 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1181 first = kFALSE;
1182 }
1183 delete arr;
c57f56b7 1184 }
149d288c 1185 arr = fOutputFiles.Tokenize(",");
c57f56b7 1186 TIter next(arr);
0f389141 1187 Bool_t first = kTRUE;
f790bc1b 1188 const char *comment = "Files to be saved";
43da816a 1189 while ((os=(TObjString*)next())) {
e1eaf596 1190 // Ignore ouputs in jdl that are also in outputarchive
1191 TString sout = os->GetString();
f790bc1b 1192 sout.ReplaceAll("*", "");
1193 sout.ReplaceAll(".root", "");
e1eaf596 1194 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1195 if (fOutputArchive.Contains(sout)) continue;
0f389141 1196 if (!first) comment = NULL;
43da816a 1197 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1198 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1199 else
0f389141 1200 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1201 first = kFALSE;
f790bc1b 1202 if (fMergeExcludes.Contains(sout)) continue;
1203 if (!os->GetString().Contains("@") && fCloseSE.Length())
1204 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1205 else
1206 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1207 }
c57f56b7 1208 delete arr;
0f389141 1209 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1210 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1211 TString validationScript = fValidationScript;
0f389141 1212 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1213 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1214 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1215 if (fMasterResubmitThreshold) {
1216 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1217 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1218 }
d2a409b2 1219 // Write a jdl with 2 input parameters: collection name and output dir name.
1220 WriteJDL(copy);
c57f56b7 1221 }
1222 // Copy jdl to grid workspace
a8739e8a 1223 if (copy) {
b5e4aaa7 1224 // Check if an output directory was defined and valid
1225 if (!fGridOutputDir.Length()) {
1226 Error("CreateJDL", "You must define AliEn output directory");
1227 return kFALSE;
1228 } else {
1229 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1230 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1231 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1232 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1233 } else {
1234 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1235 return kFALSE;
1236 }
1237 }
1238 gGrid->Cd(workdir);
1239 }
648174cf 1240 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1241 TString mergeJDLName = fExecutable;
1242 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1243 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1244 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1245 if (fProductionMode) {
648174cf 1246 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1247 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1248 }
648174cf 1249 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1250 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1251 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1252 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1253 if (fMergeViaJDL) {
1254 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1255 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1256 }
648174cf 1257 }
c57f56b7 1258 if (fAdditionalLibs.Length()) {
e7c71df0 1259 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1260 TObjString *os;
1261 TIter next(arr);
1262 while ((os=(TObjString*)next())) {
c57f56b7 1263 if (os->GetString().Contains(".so")) continue;
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1265 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1266 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1267 }
1268 delete arr;
1269 }
4e5c5506 1270 if (fPackages) {
1271 TIter next(fPackages);
1272 TObject *obj;
1273 while ((obj=next())) {
fdbbc7be 1274 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1275 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1276 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1277 }
1278 }
c57f56b7 1279 }
1280 return kTRUE;
1281}
1282
a8739e8a 1283//______________________________________________________________________________
d2a409b2 1284Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1285{
1286// Writes one or more JDL's corresponding to findex. If findex is negative,
1287// all run numbers are considered in one go (jdl). For non-negative indices
1288// they correspond to the indices in the array fInputFiles.
1289 if (!fInputFiles) return kFALSE;
d3b18c4c 1290 TObject *os;
1291 TString workdir;
1292 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1293 workdir += fGridWorkingDir;
f47d5cb4 1294 TString stageName = "$2";
1295 if (fProductionMode) stageName = "$4";
1296 if (!fMergeDirName.IsNull()) {
1297 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1298 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1299 } else {
1300 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1301 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1302 }
d3b18c4c 1303 if (fProductionMode) {
a8739e8a 1304 TIter next(fInputFiles);
b3e07543 1305 while ((os=next())) {
d3b18c4c 1306 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1307 }
d3b18c4c 1308 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1309 } else {
1310 if (!fRunNumbers.Length() && !fRunRange[0]) {
1311 // One jdl with no parameters in case input data is specified by name.
1312 TIter next(fInputFiles);
1313 while ((os=next()))
1314 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1315 if (!fOutputSingle.IsNull())
1316 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1317 else {
1318 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1319 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1320 }
1321 } else {
1322 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1323 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1324 if (!fOutputSingle.IsNull()) {
1325 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1326 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1327 } else {
1328 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1329 }
1330 }
a8739e8a 1331 }
1332
a8739e8a 1333 // Generate the JDL as a string
1334 TString sjdl = fGridJDL->Generate();
0f389141 1335 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1336 // Final merge jdl
1337 if (!fMergeDirName.IsNull()) {
1338 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1339 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1340 } else {
1341 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1342 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1343 }
b3e07543 1344 TString sjdl2 = fMergingJDL->Generate();
1345 Int_t index, index1;
a8739e8a 1346 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1347 sjdl.ReplaceAll("(member", "\n (member");
1348 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1349 sjdl.ReplaceAll("{", "{\n ");
1350 sjdl.ReplaceAll("};", "\n};");
1351 sjdl.ReplaceAll("{\n \n", "{\n");
1352 sjdl.ReplaceAll("\n\n", "\n");
1353 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1354 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1355 sjdl1.ReplaceAll("(member", "\n (member");
1356 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1357 sjdl1.ReplaceAll("{", "{\n ");
1358 sjdl1.ReplaceAll("};", "\n};");
1359 sjdl1.ReplaceAll("{\n \n", "{\n");
1360 sjdl1.ReplaceAll("\n\n", "\n");
1361 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1362 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1363 sjdl2.ReplaceAll("(member", "\n (member");
1364 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1365 sjdl2.ReplaceAll("{", "{\n ");
1366 sjdl2.ReplaceAll("};", "\n};");
1367 sjdl2.ReplaceAll("{\n \n", "{\n");
1368 sjdl2.ReplaceAll("\n\n", "\n");
1369 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1370 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1371 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1372 index = sjdl.Index("JDLVariables");
1373 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1374 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1375 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1376 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1377 index = fJobTag.Index(":");
1378 if (index < 0) index = fJobTag.Length();
1379 TString jobTag = fJobTag;
f47d5cb4 1380 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1381 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1382 if (fProductionMode) {
1383 sjdl1.Prepend("# Generated merging jdl (production mode) \
1384 \n# $1 = full alien path to output directory to be merged \
1385 \n# $2 = train number \
1386 \n# $3 = production (like LHC10b) \
1387 \n# $4 = merging stage \
1388 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1389 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1390 sjdl2.Prepend("# Generated merging jdl \
1391 \n# $1 = full alien path to output directory to be merged \
1392 \n# $2 = train number \
1393 \n# $3 = production (like LHC10b) \
1394 \n# $4 = merging stage \
1395 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1396 } else {
1397 sjdl1.Prepend("# Generated merging jdl \
1398 \n# $1 = full alien path to output directory to be merged \
1399 \n# $2 = merging stage \
03335324 1400 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1401 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1402 sjdl2.Prepend("# Generated merging jdl \
1403 \n# $1 = full alien path to output directory to be merged \
1404 \n# $2 = merging stage \
03335324 1405 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1406 }
0f389141 1407 index = sjdl1.Index("JDLVariables");
1408 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1409 index = sjdl2.Index("JDLVariables");
1410 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1411 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1412 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1413 index = sjdl2.Index("Split =");
1414 if (index>=0) {
1415 index1 = sjdl2.Index("\n", index);
1416 sjdl2.Remove(index, index1-index+1);
1417 }
1418 index = sjdl2.Index("SplitMaxInputFileNumber");
1419 if (index>=0) {
1420 index1 = sjdl2.Index("\n", index);
1421 sjdl2.Remove(index, index1-index+1);
1422 }
1423 index = sjdl2.Index("InputDataCollection");
1424 if (index>=0) {
1425 index1 = sjdl2.Index(";", index);
1426 sjdl2.Remove(index, index1-index+1);
1427 }
1428 index = sjdl2.Index("InputDataListFormat");
1429 if (index>=0) {
1430 index1 = sjdl2.Index("\n", index);
1431 sjdl2.Remove(index, index1-index+1);
1432 }
1433 index = sjdl2.Index("InputDataList");
1434 if (index>=0) {
1435 index1 = sjdl2.Index("\n", index);
1436 sjdl2.Remove(index, index1-index+1);
1437 }
f47d5cb4 1438 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1439 // Write jdl to file
a8739e8a 1440 ofstream out;
d2a409b2 1441 out.open(fJDLName.Data(), ios::out);
a8739e8a 1442 if (out.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1444 return kFALSE;
1445 }
1446 out << sjdl << endl;
b3e07543 1447 out.close();
0f389141 1448 TString mergeJDLName = fExecutable;
1449 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1450 if (fMergeViaJDL) {
1451 ofstream out1;
1452 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1453 if (out1.bad()) {
d3b18c4c 1454 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1455 return kFALSE;
1456 }
1457 out1 << sjdl1 << endl;
b3e07543 1458 out1.close();
1459 ofstream out2;
1460 TString finalJDL = mergeJDLName;
1461 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1462 out2.open(finalJDL.Data(), ios::out);
1463 if (out2.bad()) {
1464 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1465 return kFALSE;
1466 }
1467 out2 << sjdl2 << endl;
1468 out2.close();
0f389141 1469 }
a8739e8a 1470
1471 // Copy jdl to grid workspace
1472 if (!copy) {
d3b18c4c 1473 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1474 } else {
d2a409b2 1475 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1476 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1477 TString finalJDL = mergeJDLName;
1478 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1479 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1480 if (fProductionMode) {
b5fe9cba 1481 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1482 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1483 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1484 }
d2a409b2 1485 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1486 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1487 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1488 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1489 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1490 if (fMergeViaJDL) {
b3e07543 1491 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1492 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1493 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1494 }
a8739e8a 1495 }
1496 return kTRUE;
1497}
1498
c57f56b7 1499//______________________________________________________________________________
5513444a 1500Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1501{
1502// Returns true if file exists.
5513444a 1503 if (!gGrid) return kFALSE;
2ea55496 1504 TString slfn = lfn;
1505 slfn.ReplaceAll("alien://","");
1506 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1507 if (!res) return kFALSE;
1508 TMap *map = dynamic_cast<TMap*>(res->At(0));
1509 if (!map) {
1510 delete res;
1511 return kFALSE;
1512 }
1513 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1514 if (!objs || !objs->GetString().Length()) {
1515 delete res;
1516 return kFALSE;
1517 }
1518 delete res;
1519 return kTRUE;
1520}
1521
923e2ca5 1522//______________________________________________________________________________
1523Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1524{
1525// Returns true if directory exists. Can be also a path.
1526 if (!gGrid) return kFALSE;
1527 // Check if dirname is a path
1528 TString dirstripped = dirname;
1529 dirstripped = dirstripped.Strip();
1530 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1531 TString dir = gSystem->BaseName(dirstripped);
1532 dir += "/";
1533 TString path = gSystem->DirName(dirstripped);
1534 TGridResult *res = gGrid->Ls(path, "-F");
1535 if (!res) return kFALSE;
1536 TIter next(res);
1537 TMap *map;
1538 TObject *obj;
1539 while ((map=dynamic_cast<TMap*>(next()))) {
1540 obj = map->GetValue("name");
1541 if (!obj) break;
1542 if (dir == obj->GetName()) {
1543 delete res;
1544 return kTRUE;
1545 }
1546 }
1547 delete res;
1548 return kFALSE;
1549}
1550
c57f56b7 1551//______________________________________________________________________________
a2f5fc01 1552void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1553{
1554// Check input data type.
a2f5fc01 1555 isCollection = kFALSE;
1556 isXml = kFALSE;
1557 useTags = kFALSE;
c57f56b7 1558 if (!gGrid) {
1559 Error("CheckDataType", "No connection to grid");
1560 return;
1561 }
a2f5fc01 1562 isCollection = IsCollection(lfn);
c57f56b7 1563 TString msg = "\n##### file: ";
1564 msg += lfn;
a2f5fc01 1565 if (isCollection) {
c57f56b7 1566 msg += " type: raw_collection;";
1567 // special treatment for collections
a2f5fc01 1568 isXml = kFALSE;
c57f56b7 1569 // check for tag files in the collection
1570 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1571 if (!res) {
1572 msg += " using_tags: No (unknown)";
23329835 1573 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1574 return;
1575 }
1576 const char* typeStr = res->GetKey(0, "origLFN");
1577 if (!typeStr || !strlen(typeStr)) {
1578 msg += " using_tags: No (unknown)";
23329835 1579 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1580 return;
1581 }
1582 TString file = typeStr;
a2f5fc01 1583 useTags = file.Contains(".tag");
1584 if (useTags) msg += " using_tags: Yes";
c57f56b7 1585 else msg += " using_tags: No";
23329835 1586 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1587 return;
1588 }
1589 TString slfn(lfn);
1590 slfn.ToLower();
a2f5fc01 1591 isXml = slfn.Contains(".xml");
1592 if (isXml) {
c57f56b7 1593 // Open xml collection and check if there are tag files inside
1594 msg += " type: xml_collection;";
1595 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1596 if (!coll) {
1597 msg += " using_tags: No (unknown)";
23329835 1598 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1599 return;
1600 }
1601 TMap *map = coll->Next();
1602 if (!map) {
1603 msg += " using_tags: No (unknown)";
23329835 1604 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1605 return;
1606 }
1607 map = (TMap*)map->GetValue("");
1608 TString file;
1609 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1610 useTags = file.Contains(".tag");
c57f56b7 1611 delete coll;
a2f5fc01 1612 if (useTags) msg += " using_tags: Yes";
c57f56b7 1613 else msg += " using_tags: No";
23329835 1614 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1615 return;
1616 }
a2f5fc01 1617 useTags = slfn.Contains(".tag");
c57f56b7 1618 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1619 else msg += " type: unknown file;";
a2f5fc01 1620 if (useTags) msg += " using_tags: Yes";
c57f56b7 1621 else msg += " using_tags: No";
23329835 1622 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1623}
1624
4e5c5506 1625//______________________________________________________________________________
1626void AliAnalysisAlien::EnablePackage(const char *package)
1627{
1628// Enables a par file supposed to exist in the current directory.
1629 TString pkg(package);
1630 pkg.ReplaceAll(".par", "");
1631 pkg += ".par";
1632 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1633 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1634 return;
1635 }
1636 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1637 Info("EnablePackage", "AliEn plugin will use .par packages");
1638 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1639 if (!fPackages) {
1640 fPackages = new TObjArray();
1641 fPackages->SetOwner();
1642 }
1643 fPackages->Add(new TObjString(pkg));
1644}
1645
3bdcb562 1646//______________________________________________________________________________
1647TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1648{
1649// Make a tree from files having the location specified in fFileForTestMode.
1650// Inspired from JF's CreateESDChain.
1651 if (fFileForTestMode.IsNull()) {
1652 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1653 return NULL;
1654 }
1655 if (gSystem->AccessPathName(fFileForTestMode)) {
1656 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1657 return NULL;
1658 }
1659 // Open the file
1660 ifstream in;
1661 in.open(fFileForTestMode);
1662 Int_t count = 0;
1663 // Read the input list of files and add them to the chain
1664 TString line;
1665 TChain *chain = new TChain(treeName);
1666 while (in.good())
1667 {
1668 in >> line;
1669 if (line.IsNull()) continue;
1670 if (count++ == fNtestFiles) break;
1671 TString esdFile(line);
1672 TFile *file = TFile::Open(esdFile);
1673 if (file) {
1674 if (!file->IsZombie()) chain->Add(esdFile);
1675 file->Close();
1676 } else {
1677 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1678 }
1679 }
1680 in.close();
1681 if (!chain->GetListOfFiles()->GetEntries()) {
1682 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1683 delete chain;
1684 return NULL;
1685 }
1686// chain->ls();
1687 return chain;
1688}
1689
d2a409b2 1690//______________________________________________________________________________
1691const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1692{
1693// Get job status for all jobs with jobid>jobidstart.
1694 static char mstatus[20];
1695 mstatus[0] = '\0';
1696 nrunning = 0;
1697 nwaiting = 0;
1698 nerror = 0;
1699 ndone = 0;
1700 TGridJobStatusList *list = gGrid->Ps("");
1701 if (!list) return mstatus;
1702 Int_t nentries = list->GetSize();
1703 TGridJobStatus *status;
1704 Int_t pid;
1705 for (Int_t ijob=0; ijob<nentries; ijob++) {
1706 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1707 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1708 if (pid<jobidstart) continue;
1709 if (pid == lastid) {
8ddad121 1710 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1711 }
1712 switch (status->GetStatus()) {
1713 case TGridJobStatus::kWAITING:
1714 nwaiting++; break;
1715 case TGridJobStatus::kRUNNING:
1716 nrunning++; break;
1717 case TGridJobStatus::kABORTED:
1718 case TGridJobStatus::kFAIL:
1719 case TGridJobStatus::kUNKNOWN:
1720 nerror++; break;
1721 case TGridJobStatus::kDONE:
1722 ndone++;
1723 }
1724 }
1725 list->Delete();
1726 delete list;
1727 return mstatus;
1728}
1729
c57f56b7 1730//______________________________________________________________________________
1731Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1732{
1733// Returns true if file is a collection. Functionality duplicated from
1734// TAlien::Type() because we don't want to directly depend on TAlien.
1735 if (!gGrid) {
1736 Error("IsCollection", "No connection to grid");
1737 return kFALSE;
1738 }
1739 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1740 if (!res) return kFALSE;
1741 const char* typeStr = res->GetKey(0, "type");
1742 if (!typeStr || !strlen(typeStr)) return kFALSE;
1743 if (!strcmp(typeStr, "collection")) return kTRUE;
1744 delete res;
1745 return kFALSE;
1746}
1747
fe2d7fc2 1748//______________________________________________________________________________
1749Bool_t AliAnalysisAlien::IsSingleOutput() const
1750{
1751// Check if single-ouput option is on.
1752 return (!fOutputSingle.IsNull());
1753}
1754
16a4353c 1755//______________________________________________________________________________
1756void AliAnalysisAlien::Print(Option_t *) const
1757{
1758// Print current plugin settings.
84fcd93f 1759 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1760 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1761 if (mgr && mgr->IsProofMode()) {
1762 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1763 if (TestBit(AliAnalysisGrid::kTest))
1764 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1765 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1766 if (!fProofDataSet.IsNull())
1767 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1768 if (fProofReset==1)
1769 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1770 if (fProofReset>1)
1771 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1772 if (!fRootVersionForProof.IsNull())
1773 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1774 else
1775 printf("= ROOT version requested________________________ default\n");
1776 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1777 if (!fAliRootMode.IsNull())
1778 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1779 if (fNproofWorkers)
1780 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1781 if (fNproofWorkersPerSlave)
1782 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1783 if (TestSpecialBit(kClearPackages))
1784 printf("= ClearPackages requested...\n");
1785 if (fIncludePath.Data())
1786 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1787 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1788 if (fPackages && fPackages->GetEntries()) {
1789 TIter next(fPackages);
1790 TObject *obj;
1791 TString list;
1792 while ((obj=next())) list += obj->GetName();
1793 printf("= Par files to be used: ________________________ %s\n", list.Data());
1794 }
1795 if (TestSpecialBit(kProofConnectGrid))
1796 printf("= Requested PROOF connection to grid\n");
1797 return;
1798 }
e1c22e21 1799 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1800 if (fOverwriteMode) {
1801 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1802 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1803 }
348be253 1804 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1805 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1806 printf("= Production mode:______________________________ %d\n", fProductionMode);
1807 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1808 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1809 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1810 if (fUser.Length())
84fcd93f 1811 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1812 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1813 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1814 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1815 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1816 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1817 if (fRunNumbers.Length())
84fcd93f 1818 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1819 if (fRunRange[0])
74013535 1820 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1821 if (!fRunRange[0] && !fRunNumbers.Length()) {
1822 TIter next(fInputFiles);
1823 TObject *obj;
1824 TString list;
1825 while ((obj=next())) list += obj->GetName();
84fcd93f 1826 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1827 }
1828 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1829 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1830 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1831 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1832 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1833 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1834 printf("=====================================================================\n");
1835 printf("= Job price: ___________________________________ %d\n", fPrice);
1836 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1837 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1838 if (fMaxInitFailed>0)
84fcd93f 1839 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1840 if (fMasterResubmitThreshold>0)
84fcd93f 1841 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1842 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1843 if (fNrunsPerMaster>0)
84fcd93f 1844 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1845 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1846 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1847 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1848 if (fArguments.Length())
84fcd93f 1849 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1850 if (fExecutableArgs.Length())
1851 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1852 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1853 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1854 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1855 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1856 if (fDatasetName)
84fcd93f 1857 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1858 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1859 if (fIncludePath.Data())
84fcd93f 1860 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1861 if (fCloseSE.Length())
84fcd93f 1862 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1863 if (fFriendChainName.Length())
84fcd93f 1864 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1865 if (fPackages && fPackages->GetEntries()) {
16a4353c 1866 TIter next(fPackages);
1867 TObject *obj;
1868 TString list;
1869 while ((obj=next())) list += obj->GetName();
84fcd93f 1870 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1871 }
1872}
1873
c57f56b7 1874//______________________________________________________________________________
1875void AliAnalysisAlien::SetDefaults()
1876{
1877// Set default values for everything. What cannot be filled will be left empty.
1878 if (fGridJDL) delete fGridJDL;
1879 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1880 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1881 fPrice = 1;
1882 fTTL = 30000;
1883 fSplitMaxInputFileNumber = 100;
1884 fMaxInitFailed = 0;
1885 fMasterResubmitThreshold = 0;
bb885a9e 1886 fNtestFiles = 10;
149d288c 1887 fNreplicas = 2;
a8739e8a 1888 fRunRange[0] = 0;
1889 fRunRange[1] = 0;
8e3bd06a 1890 fRunPrefix = "%d";
319593fb 1891 fNrunsPerMaster = 1;
16a4353c 1892 fMaxMergeFiles = 100;
c57f56b7 1893 fRunNumbers = "";
1894 fExecutable = "analysis.sh";
0a1c1f7f 1895 fExecutableCommand = "root -b -q";
c57f56b7 1896 fArguments = "";
631c0b05 1897 fExecutableArgs = "";
c57f56b7 1898 fAnalysisMacro = "myAnalysis.C";
1899 fAnalysisSource = "";
1900 fAdditionalLibs = "";
1901 fSplitMode = "se";
1902 fAPIVersion = "";
1903 fROOTVersion = "";
1904 fAliROOTVersion = "";
1905 fUser = ""; // Your alien user name
1906 fGridWorkingDir = "";
1907 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1908 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1909 fFriendChainName = "";
c57f56b7 1910 fGridOutputDir = "output";
b83f84f0 1911 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1912 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1913 fInputFormat = "xml-single";
1914 fJDLName = "analysis.jdl";
c6cb3634 1915 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1916 fMergeExcludes = "";
0f389141 1917 fMergeViaJDL = 0;
348be253 1918 SetUseCopy(kTRUE);
1919 SetCheckCopy(kTRUE);
149d288c 1920 SetDefaultOutputs(kTRUE);
e1c22e21 1921 fOverwriteMode = 1;
c57f56b7 1922}
1923
0f389141 1924//______________________________________________________________________________
b3e07543 1925Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1926{
b3e07543 1927// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1928 // First check if the result is already in the output directory.
1929 if (FileExists(Form("%s/%s",aliendir,filename))) {
1930 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1931 return kFALSE;
1932 }
b3e07543 1933 // Now check the last stage done.
1934 Int_t stage = 0;
1935 while (1) {
1936 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1937 stage++;
1938 }
1939 // Next stage of merging
1940 stage++;
1941 TString pattern = "*root_archive.zip";
1942 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1943 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1944 if (res) delete res;
1945 // Write standard output to file
1946 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1947 // Count the number of files inside
1948 ifstream ifile;
1949 ifile.open(Form("Stage_%d.xml",stage));
1950 if (!ifile.good()) {
1951 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1952 return kFALSE;
b3e07543 1953 }
1954 TString line;
1955 Int_t nfiles = 0;
1956 while (!ifile.eof()) {
1957 ifile >> line;
1958 if (line.Contains("/event")) nfiles++;
7c2cd90a 1959 }
b3e07543 1960 ifile.close();
1961 if (!nfiles) {
1962 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1963 return kFALSE;
1964 } else {
1965 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1966 }
b3e07543 1967 // Copy the file in the output directory
1968 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1969 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1970 // Check if this is the last stage to be done.
1971 Bool_t laststage = (nfiles<nperchunk);
1972 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1973 if (laststage) {
1974 printf("### Submiting final merging stage %d\n", stage);
1975 TString finalJDL = jdl;
1976 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1977 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1978 Int_t jobId = SubmitSingleJob(query);
1979 if (!jobId) return kFALSE;
1980 } else {
1981 printf("### Submiting merging stage %d\n", stage);
03335324 1982 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1983 Int_t jobId = SubmitSingleJob(query);
1984 if (!jobId) return kFALSE;
7c2cd90a 1985 }
b3e07543 1986 return kTRUE;
1987}
7c2cd90a 1988
1989//______________________________________________________________________________
1990Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1991{
1992// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1993 if (!gGrid) return 0;
1994 printf("=> %s ------> ",query);
1995 TGridResult *res = gGrid->Command(query);
1996 if (!res) return 0;
1997 TString jobId = res->GetKey(0,"jobId");
1998 delete res;
1999 if (jobId.IsNull()) {
2000 printf("submission failed. Reason:\n");
2001 gGrid->Stdout();
2002 gGrid->Stderr();
2003 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2004 return 0;
2005 }
2006 printf(" Job id: %s\n", jobId.Data());
2007 return atoi(jobId);
2008}
2009
2010//______________________________________________________________________________
b3e07543 2011Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2012{
b3e07543 2013// Merge given output files from basedir. Basedir can be an alien output directory
2014// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2015// files in a group (ignored for xml input). Merging can be done in stages:
2016// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2017// stage=1 : works with an xml of all root_archive.zip in the output directory
2018// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2019 TString outputFile = output;
0f389141 2020 TString command;
a2f5fc01 2021 TString outputChunk;
2022 TString previousChunk = "";
b3e07543 2023 TObjArray *listoffiles = new TObjArray();
2024// listoffiles->SetOwner();
a2f5fc01 2025 Int_t countChunk = 0;
2026 Int_t countZero = nmaxmerge;
0f389141 2027 Bool_t merged = kTRUE;
a2f5fc01 2028 Int_t index = outputFile.Index("@");
2029 if (index > 0) outputFile.Remove(index);
7c2cd90a 2030 TString inputFile = outputFile;
b3e07543 2031 TString sbasedir = basedir;
2032 if (sbasedir.Contains(".xml")) {
2033 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2034 nmaxmerge = 9999999;
2035 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2036 if (!coll) {
2037 ::Error("MergeOutput", "Input XML collection empty.");
2038 return kFALSE;
2039 }
2040 // Iterate grid collection
2041 while (coll->Next()) {
2042 TString fname = gSystem->DirName(coll->GetTURL());
2043 fname += "/";
2044 fname += inputFile;
2045 listoffiles->Add(new TNamed(fname.Data(),""));
2046 }
2047 } else {
2048 command = Form("find %s/ *%s", basedir, inputFile.Data());
2049 printf("command: %s\n", command.Data());
2050 TGridResult *res = gGrid->Command(command);
2051 if (!res) {
2052 ::Error("MergeOutput","No result for the find command\n");
2053 delete listoffiles;
2054 return kFALSE;
2055 }
2056 TIter nextmap(res);
2057 TMap *map = 0;
2058 while ((map=(TMap*)nextmap())) {
2059 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2060 if (!objs || !objs->GetString().Length()) {
2061 // Nothing found - skip this output
2062 delete res;
2063 delete listoffiles;
2064 return kFALSE;
2065 }
2066 listoffiles->Add(new TNamed(objs->GetName(),""));
2067 }
2068 delete res;
2069 }
2070 if (!listoffiles->GetEntries()) {
7c2cd90a 2071 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2072 delete listoffiles;
0f389141 2073 return kFALSE;
2074 }
2075
2076 TFileMerger *fm = 0;
b3e07543 2077 TIter next0(listoffiles);
2078 TObjArray *listoffilestmp = new TObjArray();
2079 listoffilestmp->SetOwner();
2080 TObject *nextfile;
2081 TString snextfile;
2082 // Keep only the files at upper level
2083 Int_t countChar = 0;
2084 while ((nextfile=next0())) {
2085 snextfile = nextfile->GetName();
2086 Int_t crtCount = snextfile.CountChar('/');
2087 if (nextfile == listoffiles->First()) countChar = crtCount;
2088 if (crtCount < countChar) countChar = crtCount;
2089 }
2090 next0.Reset();
2091 while ((nextfile=next0())) {
2092 snextfile = nextfile->GetName();
2093 Int_t crtCount = snextfile.CountChar('/');
2094 if (crtCount > countChar) {
2095 delete nextfile;
2096 continue;
2097 }
2098 listoffilestmp->Add(nextfile);
2099 }
2100 delete listoffiles;
2101 listoffiles = listoffilestmp; // Now contains 'good' files
2102 listoffiles->Print();
2103 TIter next(listoffiles);
7c2cd90a 2104 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2105 outputChunk = outputFile;
2106 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2107 // Check for existent temporary merge files
e1c22e21 2108 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2109 // Preserve old merging functionality for stage 0.
2110 if (stage==0) {
2111 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2112 while (1) {
2113 // Skip as many input files as in a chunk
2ea55496 2114 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2115 nextfile = next();
2116 if (!nextfile) {
2ea55496 2117 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2118 delete listoffiles;
2ea55496 2119 return kFALSE;
2120 }
b3e07543 2121 snextfile = nextfile->GetName();
7c2cd90a 2122 }
2123 outputChunk = outputFile;
2124 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2125 countChunk++;
2126 if (gSystem->AccessPathName(outputChunk)) continue;
2127 // Merged file with chunks up to <countChunk> found
2128 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2129 previousChunk = outputChunk;
2130 break;
2131 }
2132 }
2133 countZero = nmaxmerge;
2134
b3e07543 2135 while ((nextfile=next())) {
2136 snextfile = nextfile->GetName();
2ea55496 2137 // Loop 'find' results and get next LFN
7c2cd90a 2138 if (countZero == nmaxmerge) {
2139 // First file in chunk - create file merger and add previous chunk if any.
2140 fm = new TFileMerger(kFALSE);
2141 fm->SetFastMethod(kTRUE);
2142 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2143 outputChunk = outputFile;
2144 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2145 }
2146 // If last file found, put merged results in the output file
b3e07543 2147 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2148 // Add file to be merged and decrement chunk counter.
b3e07543 2149 fm->AddFile(snextfile);
7c2cd90a 2150 countZero--;
b3e07543 2151 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2152 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2153 // Nothing found - skip this output
2154 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2155 merged = kFALSE;
2156 break;
7c2cd90a 2157 }
2158 fm->OutputFile(outputChunk);
2159 // Merge the outputs, then go to next chunk
2160 if (!fm->Merge()) {
2161 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2162 merged = kFALSE;
2163 break;
7c2cd90a 2164 } else {
2165 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2166 gSystem->Unlink(previousChunk);
2167 }
b3e07543 2168 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2169 countChunk++;
2170 countZero = nmaxmerge;
2171 previousChunk = outputChunk;
0f389141 2172 }
0f389141 2173 }
b3e07543 2174 delete listoffiles;
2175 delete fm;
7c2cd90a 2176 return merged;
2177 }
2178 // Merging stage different than 0.
2179 // Move to the begining of the requested chunk.
7c2cd90a 2180 fm = new TFileMerger(kFALSE);
2181 fm->SetFastMethod(kTRUE);
b3e07543 2182 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2183 delete listoffiles;
7c2cd90a 2184 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2185 // Nothing found - skip this output
2186 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2187 delete fm;
2188 return kFALSE;
2189 }
b3e07543 2190 fm->OutputFile(outputFile);
7c2cd90a 2191 // Merge the outputs
2192 if (!fm->Merge()) {
2193 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2194 delete fm;
2195 return kFALSE;
2196 } else {
b3e07543 2197 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2198 }
7c2cd90a 2199 delete fm;
2200 return kTRUE;
0f389141 2201}
2202
c57f56b7 2203//______________________________________________________________________________
2204Bool_t AliAnalysisAlien::MergeOutputs()
2205{
2206// Merge analysis outputs existing in the AliEn space.
2207 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2208 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2209 if (!Connect()) {
2210 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2211 return kFALSE;
0f389141 2212 }
c9e8f7fd 2213 if (fMergeViaJDL) {
2214 if (!TestBit(AliAnalysisGrid::kMerge)) {
2215 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2216 return kFALSE;
2217 }
2218 if (fProductionMode) {
2219 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2220 return kFALSE;
2221 }
0f389141 2222 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2223 if (!SubmitMerging()) return kFALSE;
0f389141 2224 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2225 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2226 return kFALSE;
c57f56b7 2227 }
2228 // Get the output path
dc030a24 2229 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2230 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2231 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2232 return kFALSE;
2233 }
2234 if (!fOutputFiles.Length()) {
2235 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2236 return kFALSE;
0f389141 2237 }
2238 // Check if fast read option was requested
e1c22e21 2239 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2240 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2241 if (fFastReadOption) {
d3339be3 2242 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2243 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2244 gEnv->SetValue("XNet.ConnectTimeout",50);
2245 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2246 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2247 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2248 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2249 }
e8b839ab 2250 // Make sure we change the temporary directory
2251 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2252 // Set temporary compilation directory to current one
2253 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2254 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2255 TIter next(list);
2256 TObjString *str;
a2f5fc01 2257 TString outputFile;
c57f56b7 2258 Bool_t merged = kTRUE;
2259 while((str=(TObjString*)next())) {
a2f5fc01 2260 outputFile = str->GetString();
2261 Int_t index = outputFile.Index("@");
2262 if (index > 0) outputFile.Remove(index);
2263 TString outputChunk = outputFile;
2264 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2265 // Skip already merged outputs
a2f5fc01 2266 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2267 if (fOverwriteMode) {
a2f5fc01 2268 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2269 gSystem->Unlink(outputFile);
2270 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2271 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2272 outputChunk.Data());
2273 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2274 }
2275 } else {
a2f5fc01 2276 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2277 continue;
2278 }
2279 } else {
a2f5fc01 2280 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2281 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2282 outputChunk.Data());
2283 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2284 }
2285 }
bb885a9e 2286 if (fMergeExcludes.Length() &&
a2f5fc01 2287 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2288 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2289 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2290 if (!merged) {
2291 Error("MergeOutputs", "Terminate() will NOT be executed");
2292 return kFALSE;
ff07ec61 2293 }
2294 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2295 if (fileOpened) fileOpened->Close();
c57f56b7 2296 }
0f389141 2297 return kTRUE;
c57f56b7 2298}
2299
bb885a9e 2300//______________________________________________________________________________
2301void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2302{
2303// Use the output files connected to output containers from the analysis manager
2304// rather than the files defined by SetOutputFiles
2305 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2306 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2307 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2308}
2309
149d288c 2310//______________________________________________________________________________
2311void AliAnalysisAlien::SetOutputFiles(const char *list)
2312{
2313// Manually set the output files list.
2314// Removes duplicates. Not allowed if default outputs are not disabled.
2315 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2316 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2317 return;
2318 }
2319 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2320 fOutputFiles = "";
2321 TString slist = list;
2322 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2323 TObjArray *arr = slist.Tokenize(" ");
2324 TObjString *os;
2325 TIter next(arr);
2326 TString sout;
2327 while ((os=(TObjString*)next())) {
2328 sout = os->GetString();
2329 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2330 if (fOutputFiles.Contains(sout)) continue;
2331 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2332 fOutputFiles += sout;
2333 }
2334 delete arr;
f790bc1b 2335}
149d288c 2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetOutputArchive(const char *list)
2339{
2340// Manually set the output archive list. Free text - you are on your own...
2341// Not allowed if default outputs are not disabled.
2342 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2343 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2344 return;
2345 }
2346 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2347 fOutputArchive = list;
2348}
2349
2350//______________________________________________________________________________
2351void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2352{
2353// Setting a prefered output SE is not allowed anymore.
2354 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2355}
2356
5513444a 2357//______________________________________________________________________________
2358Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2359{
2360// Start remote grid analysis.
3bdcb562 2361 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2362 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2363 if (!mgr || !mgr->IsInitialized()) {
2364 Error("StartAnalysis", "You need an initialized analysis manager for this");
2365 return kFALSE;
2366 }
2367 // Are we in PROOF mode ?
2368 if (mgr->IsProofMode()) {
90d50a8c 2369 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2370 if (fProofCluster.IsNull()) {
2371 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2372 return kFALSE;
2373 }
2374 if (fProofDataSet.IsNull() && !testMode) {
2375 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2376 return kFALSE;
2377 }
2378 // Set the needed environment
2379 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2380 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2381 if (fProofReset && !testMode) {
2382 if (fProofReset==1) {
2383 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2384 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2385 } else {
2386 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2387 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2388 }
2389 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2390 return kFALSE;
2391 }
2392 // Do we need to change the ROOT version ? The success of this cannot be checked.
2393 if (!fRootVersionForProof.IsNull() && !testMode) {
2394 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2395 fProofCluster.Data(), fRootVersionForProof.Data()));
2396 }
2397 // Connect to PROOF and check the status
2398 Long_t proof = 0;
27734f0e 2399 TString sworkers;
2400 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2401 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2402 if (!testMode) {
27734f0e 2403 if (!sworkers.IsNull())
2404 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2405 else
2406 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2407 } else {
2408 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2409 if (!proof) {
2410 Error("StartAnalysis", "Could not start PROOF in test mode");
2411 return kFALSE;
2412 }
2413 }
2414 if (!proof) {
2415 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2416 return kFALSE;
2417 }
27734f0e 2418 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2419 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2420 // Is dataset existing ?
2421 if (!testMode) {
2422 TString dataset = fProofDataSet;
2423 Int_t index = dataset.Index("#");
2424 if (index>=0) dataset.Remove(index);
830acc4c 2425// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2426// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2427// return kFALSE;
2428// }
2429// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2430 }
2431 // Is ClearPackages() needed ?
2432 if (TestSpecialBit(kClearPackages)) {
2433 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2434 gROOT->ProcessLine("gProof->ClearPackages();");
2435 }
2436 // Is a given aliroot mode requested ?
2437 TList optionsList;
1ac846e9 2438 TString parLibs;
830acc4c 2439 if (!fAliRootMode.IsNull()) {
3bdcb562 2440 TString alirootMode = fAliRootMode;
2441 if (alirootMode == "default") alirootMode = "";
2442 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2443 optionsList.SetOwner();
2444 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2445 // Check the additional libs to be loaded
2446 TString extraLibs;
1ac846e9 2447 Bool_t parMode = kFALSE;
3bdcb562 2448 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2449 // Parse the extra libs for .so
2450 if (fAdditionalLibs.Length()) {
2451 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2452 TIter next(list);
2453 TObjString *str;
1ac846e9 2454 while((str=(TObjString*)next())) {
2455 if (str->GetString().Contains(".so")) {
2456 if (parMode) {
2457 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2458 break;
2459 }
2460 TString stmp = str->GetName();
2461 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2462 stmp.ReplaceAll(".so","");
2463 if (!extraLibs.IsNull()) extraLibs += ":";
2464 extraLibs += stmp;
2465 continue;
2466 }
2467 if (str->GetString().Contains(".par")) {
2468 // The first par file found in the list will not allow any further .so
2469 parMode = kTRUE;
2470 if (!parLibs.IsNull()) parLibs += ":";
2471 parLibs += str->GetName();
2472 continue;
2473 }
3bdcb562 2474 }
2475 if (list) delete list;
2476 }
dc030a24 2477 if (!extraLibs.IsNull()) {
2478 Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
2479 optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
2480 }
830acc4c 2481 // Check extra includes
2482 if (!fIncludePath.IsNull()) {
2483 TString includePath = fIncludePath;
2484 includePath.ReplaceAll(" ",":");
dc030a24 2485 includePath.ReplaceAll("$ALICE_ROOT/","");
2486 includePath.ReplaceAll("${ALICE_ROOT}/","");
8ddad121 2487 includePath.ReplaceAll("-I","");
dc030a24 2488 includePath.Remove(TString::kTrailing, ':');
830acc4c 2489 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2490 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2491 }
27734f0e 2492 // Check if connection to grid is requested
2493 if (TestSpecialBit(kProofConnectGrid))
2494 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2495 // Enable AliRoot par
2496 if (testMode) {
2497 // Enable proof lite package
2498 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2499 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2500 TNamed *obj = (TNamed*)optionsList.At(i);
2501 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2502 }
2503 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2504 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2505 Info("StartAnalysis", "AliRootProofLite enabled");
2506 } else {
2507 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2508 return kFALSE;
2509 }
2510 } else {
dc030a24 2511 if ( ! fAliROOTVersion.IsNull() ) {
2512 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2513 fAliROOTVersion.Data(), &optionsList))) {
2514 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2515 return kFALSE;
2516 }
2517 }
27734f0e 2518 }
1ac846e9 2519 // Enable first par files from fAdditionalLibs
2520 if (!parLibs.IsNull()) {
2521 TObjArray *list = parLibs.Tokenize(":");
2522 TIter next(list);
2523 TObjString *package;
2524 while((package=(TObjString*)next())) {
40501bb7 2525 TString spkg = package->GetName();
2526 spkg.ReplaceAll(".par", "");
2527 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2528 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2529 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2530 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2531 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2532 return kFALSE;
2533 }
2534 } else {
2535 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2536 return kFALSE;
2537 }
2538 }
2539 if (list) delete list;
2540 }
3bdcb562 2541 } else {
2542 if (fAdditionalLibs.Contains(".so") && !testMode) {
2543 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2544 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2545 return kFALSE;
2546 }
2547 }
2548 // Enable par files if requested
2549 if (fPackages && fPackages->GetEntries()) {
2550 TIter next(fPackages);
2551 TObject *package;
2552 while ((package=next())) {
1ac846e9 2553 // Skip packages already enabled
2554 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2555 TString spkg = package->GetName();
2556 spkg.ReplaceAll(".par", "");
2557 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
dc030a24 2558 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2559 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2560 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2561 return kFALSE;
2562 }
2563 } else {
2564 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2565 return kFALSE;
2566 }
2567 }
2568 }
2569 // Do we need to load analysis source files ?
2570 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2571 if (fAnalysisSource.Length()) {
2572 TObjArray *list = fAnalysisSource.Tokenize(" ");
2573 TIter next(list);
2574 TObjString *str;
2575 while((str=(TObjString*)next())) {
2576 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
dc030a24 2577 }
3bdcb562 2578 if (list) delete list;
2579 }
830acc4c 2580 if (testMode) {
2581 // Register dataset to proof lite.
2582 if (fFileForTestMode.IsNull()) {
2583 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2584 return kFALSE;
2585 }
2586 if (gSystem->AccessPathName(fFileForTestMode)) {
2587 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2588 return kFALSE;
2589 }
2590 TFileCollection *coll = new TFileCollection();
2591 coll->AddFromFile(fFileForTestMode);
8ddad121 2592 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2593 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2594 }
3bdcb562 2595 return kTRUE;
2596 }
c57f56b7 2597
43da816a 2598 // Check if output files have to be taken from the analysis manager
2599 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2600 // Add output files and AOD files
2601 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2602 // Add extra files registered to the analysis manager
f790bc1b 2603 TString extra = GetListOfFiles("ext");
2604 if (!extra.IsNull()) {
149d288c 2605 extra.ReplaceAll(".root", "*.root");
f790bc1b 2606 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2607 fOutputFiles += extra;
84fcd93f 2608 }
149d288c 2609 // Compose the output archive.
b83f84f0 2610 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2611 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2612 }
f7b1cbc2 2613// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2614 if (TestBit(AliAnalysisGrid::kOffline)) {
2615 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2616 \n there nor any job run. You can revise the JDL and analysis \
2617 \n macro then run the same in \"submit\" mode.");
2618 } else if (TestBit(AliAnalysisGrid::kTest)) {
2619 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2620 \n dataset.");
2621 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2622 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2623 \n space and job submitted.");
2624 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2625 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2626 if (fMergeViaJDL) CheckInputData();
5513444a 2627 return kTRUE;
c57f56b7 2628 } else {
2629 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2630 }
2631
348be253 2632 Print();
c57f56b7 2633 if (!Connect()) {
2634 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2635 return kFALSE;
16a4353c 2636 }
d3b18c4c 2637 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2638 if (!CheckInputData()) {
2639 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2640 return kFALSE;
c57f56b7 2641 }
d3339be3 2642 if (!CreateDataset(fDataPattern)) {
2643 TString serror;
2644 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2645 if (fRunNumbers.Length()) serror = "run numbers";
2646 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2647 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2648 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2649 return kFALSE;
2650 }
c57f56b7 2651 WriteAnalysisFile();
2652 WriteAnalysisMacro();
2653 WriteExecutable();
2654 WriteValidationScript();
0f389141 2655 if (fMergeViaJDL) {
2656 WriteMergingMacro();
2657 WriteMergeExecutable();
2658 WriteValidationScript(kTRUE);
2659 }
5513444a 2660 if (!CreateJDL()) return kFALSE;
2661 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2662 if (testMode) {
c57f56b7 2663 // Locally testing the analysis
2664 Info("StartAnalysis", "\n_______________________________________________________________________ \
2665 \n Running analysis script in a daughter shell as on a worker node \
2666 \n_______________________________________________________________________");
149d288c 2667 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2668 TIter next(list);
2669 TObjString *str;
a2f5fc01 2670 TString outputFile;
c57f56b7 2671 while((str=(TObjString*)next())) {
a2f5fc01 2672 outputFile = str->GetString();
2673 Int_t index = outputFile.Index("@");
2674 if (index > 0) outputFile.Remove(index);
2675 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2676 }
2677 delete list;
2678 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2679 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2680// gSystem->Exec("cat stdout");
5513444a 2681 return kFALSE;
c57f56b7 2682 }
5513444a 2683 // Check if submitting is managed by LPM manager
a3e84053 2684 if (fProductionMode) {
5513444a 2685 TString prodfile = fJDLName;
2686 prodfile.ReplaceAll(".jdl", ".prod");
2687 WriteProductionFile(prodfile);
2688 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2689 return kFALSE;
2690 }
a8739e8a 2691 // Submit AliEn job(s)
d2a409b2 2692 gGrid->Cd(fGridOutputDir);
a8739e8a 2693 TGridResult *res;
c57f56b7 2694 TString jobID = "";
d2a409b2 2695 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2696 // Submit a given xml or a set of runs
a8739e8a 2697 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2698 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2699 if (res) {
2700 const char *cjobId = res->GetKey(0,"jobId");
2701 if (!cjobId) {
a03be957 2702 gGrid->Stdout();
2703 gGrid->Stderr();
a8739e8a 2704 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2705 return kFALSE;
a8739e8a 2706 } else {
2707 Info("StartAnalysis", "\n_______________________________________________________________________ \
2708 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2709 \n_______________________________________________________________________",
2710 fJDLName.Data(), cjobId);
2711 jobID = cjobId;
2712 }
2713 delete res;
a03be957 2714 } else {
2715 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2716 return kFALSE;
a8739e8a 2717 }
2718 } else {
d2a409b2 2719 // Submit for a range of enumeration of runs.
a03be957 2720 if (!Submit()) return kFALSE;
c57f56b7 2721 }
a8739e8a 2722
c57f56b7 2723 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2724 \n You may exit at any time and terminate the job later using the option <terminate> \
2725 \n ##################################################################################", jobID.Data());
bb885a9e 2726 gSystem->Exec("aliensh");
5513444a 2727 return kTRUE;
c57f56b7 2728}
2729
f790bc1b 2730//______________________________________________________________________________
2731const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2732{
2733// Get a comma-separated list of output files of the requested type.
2734// Type can be (case unsensitive):
2735// aod - list of aod files (std, extensions and filters)
2736// out - list of output files connected to containers (but not aod's or extras)
2737// ext - list of extra files registered to the manager
2738// ter - list of files produced in terminate
2739 static TString files;
2740 files = "";
2741 TString stype = type;
2742 stype.ToLower();
2743 TString aodfiles, extra;
2744 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2745 if (!mgr) {
2746 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2747 return files.Data();
2748 }
2749 if (mgr->GetOutputEventHandler()) {
2750 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2751 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2752 if (!extraaod.IsNull()) {
2753 aodfiles += ",";
2754 aodfiles += extraaod;
2755 }
2756 }
2757 if (stype.Contains("aod")) {
2758 files = aodfiles;
2759 if (stype == "aod") return files.Data();
2760 }
2761 // Add output files that are not in the list of AOD files
2762 TString outputfiles = "";
2763 TIter next(mgr->GetOutputs());
2764 AliAnalysisDataContainer *output;
2765 const char *filename = 0;
2766 while ((output=(AliAnalysisDataContainer*)next())) {
2767 filename = output->GetFileName();
2768 if (!(strcmp(filename, "default"))) continue;
2769 if (outputfiles.Contains(filename)) continue;
2770 if (aodfiles.Contains(filename)) continue;
2771 if (!outputfiles.IsNull()) outputfiles += ",";
2772 outputfiles += filename;
2773 }
2774 if (stype.Contains("out")) {
2775 if (!files.IsNull()) files += ",";
2776 files += outputfiles;
2777 if (stype == "out") return files.Data();
2778 }
2779 // Add extra files registered to the analysis manager
2780 TString sextra;
2781 extra = mgr->GetExtraFiles();
2782 if (!extra.IsNull()) {
2783 extra.Strip();
2784 extra.ReplaceAll(" ", ",");
2785 TObjArray *fextra = extra.Tokenize(",");
2786 TIter nextx(fextra);
2787 TObject *obj;
2788 while ((obj=nextx())) {
2789 if (aodfiles.Contains(obj->GetName())) continue;
2790 if (outputfiles.Contains(obj->GetName())) continue;
2791 if (sextra.Contains(obj->GetName())) continue;
2792 if (!sextra.IsNull()) sextra += ",";
2793 sextra += obj->GetName();
2794 }
2795 delete fextra;
2796 if (stype.Contains("ext")) {
2797 if (!files.IsNull()) files += ",";
2798 files += sextra;
2799 }
2800 }
2801 if (stype == "ext") return files.Data();
2802 TString termfiles;
2803 if (!fTerminateFiles.IsNull()) {
2804 fTerminateFiles.Strip();
2805 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2806 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2807 TIter nextx(fextra);
2808 TObject *obj;
2809 while ((obj=nextx())) {
2810 if (aodfiles.Contains(obj->GetName())) continue;
2811 if (outputfiles.Contains(obj->GetName())) continue;
2812 if (termfiles.Contains(obj->GetName())) continue;
2813 if (sextra.Contains(obj->GetName())) continue;
2814 if (!termfiles.IsNull()) termfiles += ",";
2815 termfiles += obj->GetName();
2816 }
2817 delete fextra;
2818 }
2819 if (stype.Contains("ter")) {
2820 if (!files.IsNull() && !termfiles.IsNull()) {
2821 files += ",";
2822 files += termfiles;
2823 }
2824 }
2825 return files.Data();
2826}
2827
d2a409b2 2828//______________________________________________________________________________
a03be957 2829Bool_t AliAnalysisAlien::Submit()
d2a409b2 2830{
2831// Submit all master jobs.
2832 Int_t nmasterjobs = fInputFiles->GetEntries();
2833 Long_t tshoot = gSystem->Now();
a03be957 2834 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2835 while (fNsubmitted < nmasterjobs) {
2836 Long_t now = gSystem->Now();
2837 if ((now-tshoot)>30000) {
2838 tshoot = now;
a03be957 2839 if (!SubmitNext()) return kFALSE;
d2a409b2 2840 }
2841 }
a03be957 2842 return kTRUE;
d2a409b2 2843}
2844
0f389141 2845//______________________________________________________________________________
a03be957 2846Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2847{
2848// Submit all merging jobs.
dc030a24 2849 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 2850 gGrid->Cd(fGridOutputDir);
2851 TString mergeJDLName = fExecutable;
2852 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2853 if (!fInputFiles) {
2854 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2855 return kFALSE;
2856 }
0f389141 2857 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2858 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2859 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2860 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2861 if (fOutputToRunNo) {
2862 // The output directory is the run number
2863 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2864 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2865 } else {
17b4e9a4 2866 if (!fRunNumbers.Length() && !fRunRange[0]) {
2867 // The output directory is the grid outdir
2868 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2869 runOutDir = fGridOutputDir;
2870 } else {
2871 // The output directory is the master number in 3 digits format
2872 printf("### Submitting merging job for master <%03d>\n", i);
2873 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2874 }
7c2cd90a 2875 }
2876 // Check now the number of merging stages.
2b222f7c 2877 TObjArray *list = fOutputFiles.Tokenize(",");
2878 TIter next(list);
2879 TObjString *str;
2880 TString outputFile;
2881 while((str=(TObjString*)next())) {
2882 outputFile = str->GetString();
2883 Int_t index = outputFile.Index("@");
2884 if (index > 0) outputFile.Remove(index);
2885 if (!fMergeExcludes.Contains(outputFile)) break;
2886 }
2887 delete list;
b3e07543 2888 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2889 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 2890 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 2891 }
a03be957 2892 if (!ntosubmit) return kTRUE;
9f020952 2893 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2894 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2895 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2896 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2897 \n ################################################################################################################");
0f389141 2898 gSystem->Exec("aliensh");
a03be957 2899 return kTRUE;
0f389141 2900}
2901
d2a409b2 2902//______________________________________________________________________________
a03be957 2903Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2904{
08d5b699 2905// Submit next bunch of master jobs if the queue is free. The first master job is
2906// submitted right away, while the next will not be unless the previous was split.
2907// The plugin will not submit new master jobs if there are more that 500 jobs in
2908// waiting phase.
d2a409b2 2909 static Bool_t iscalled = kFALSE;
2910 static Int_t firstmaster = 0;
2911 static Int_t lastmaster = 0;
2912 static Int_t npermaster = 0;
a03be957 2913 if (iscalled) return kTRUE;
d2a409b2 2914 iscalled = kTRUE;
2915 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2916 Int_t ntosubmit = 0;
2917 TGridResult *res;
2918 TString jobID = "";
3bdcb562 2919 Int_t nmasterjobs = fInputFiles->GetEntries();
2920 if (!fNsubmitted) {
2921 ntosubmit = 1;
2922 if (!IsUseSubmitPolicy()) {
90d50a8c 2923 if (nmasterjobs>5)
3bdcb562 2924 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2925 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2926 ntosubmit = nmasterjobs;
2927 }
2928 } else {
d2a409b2 2929 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2930 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2931 // If last master not split, just return
a03be957 2932 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2933 // No more than 100 waiting jobs
08d5b699 2934 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2935 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2936 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2937 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2938 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2939 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2940 }
d2a409b2 2941 for (Int_t i=0; i<ntosubmit; i++) {
2942 // Submit for a range of enumeration of runs.
a03be957 2943 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2944 TString query;
cd11251e 2945 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2946 runOutDir.ReplaceAll(".xml", "");
2947 if (fOutputToRunNo)
2948 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2949 else
2950 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2951 printf("********* %s\n",query.Data());
d2a409b2 2952 res = gGrid->Command(query);
2953 if (res) {
98ca124f 2954 TString cjobId1 = res->GetKey(0,"jobId");
2955 if (!cjobId1.Length()) {
d2a409b2 2956 iscalled = kFALSE;
a03be957 2957 gGrid->Stdout();
2958 gGrid->Stderr();
2959 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2960 return kFALSE;
d2a409b2 2961 } else {
2962 Info("StartAnalysis", "\n_______________________________________________________________________ \
2963 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2964 \n_______________________________________________________________________",
98ca124f 2965 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2966 jobID += cjobId1;
2967 jobID += " ";
98ca124f 2968 lastmaster = cjobId1.Atoi();
d2a409b2 2969 if (!firstmaster) firstmaster = lastmaster;
2970 fNsubmitted++;
2971 }
2972 delete res;
a03be957 2973 } else {
2974 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2975 return kFALSE;
d2a409b2 2976 }
2977 }
2978 iscalled = kFALSE;
a03be957 2979 return kTRUE;
d2a409b2 2980}
2981
c57f56b7 2982//______________________________________________________________________________
2983void AliAnalysisAlien::WriteAnalysisFile()
2984{
f10e8481 2985// Write current analysis manager into the file <analysisFile>
2986 TString analysisFile = fExecutable;
2987 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2988 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2989 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2990 if (!mgr || !mgr->IsInitialized()) {
2991 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2992 return;
2993 }
2994 // Check analysis type
2995 TObject *handler;
2996 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2997 handler = (TObject*)mgr->GetInputEventHandler();
2998 if (handler) {
65eb22f8 2999 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
3000 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
3001 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3002 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3003 } else {
3004 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
3005 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
3006 }
c57f56b7 3007 }
3008 TDirectory *cdir = gDirectory;
f10e8481 3009 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 3010 if (file) {
e16a394c 3011 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
3012 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 3013 // Unless merging makes no sense
3014 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3015 mgr->Write();
3016 delete file;
fe2d7fc2 3017 // Enable termination for local jobs
3018 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3019 }
3020 if (cdir) cdir->cd();
f10e8481 3021 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3022 }
3023 Bool_t copy = kTRUE;
d3b18c4c 3024 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3025 if (copy) {
3026 CdWork();
3027 TString workdir = gGrid->GetHomeDirectory();
3028 workdir += fGridWorkingDir;
d3b18c4c 3029 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3030 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3031 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3032 }
3033}
3034
3035//______________________________________________________________________________
3036void AliAnalysisAlien::WriteAnalysisMacro()
3037{
3038// Write the analysis macro that will steer the analysis in grid mode.
3039 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3040 ofstream out;
3041 out.open(fAnalysisMacro.Data(), ios::out);
3042 if (!out.good()) {
3043 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3044 return;
3045 }
5b273635 3046 Bool_t hasSTEERBase = kFALSE;
3047 Bool_t hasESD = kFALSE;
3048 Bool_t hasAOD = kFALSE;
3049 Bool_t hasANALYSIS = kFALSE;
3050 Bool_t hasANALYSISalice = kFALSE;
3051 Bool_t hasCORRFW = kFALSE;
c57f56b7 3052 TString func = fAnalysisMacro;
3053 TString type = "ESD";
3054 TString comment = "// Analysis using ";
c85cfc0f 3055 if (IsUseMCchain()) {
3056 type = "MC";
3057 comment += "MC";
3058 } else {
3059 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3060 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3061 type = "AOD";
3062 comment += "AOD";
3063 }
3064 }
0df6ccf2 3065 if (type!="AOD" && fFriendChainName!="") {
3066 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3067 return;
3068 }
c57f56b7 3069 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3070 else comment += " data";
3071 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3072 func.ReplaceAll(".C", "");
3073 out << "void " << func.Data() << "()" << endl;
3074 out << "{" << endl;
3075 out << comment.Data() << endl;
3076 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3077 out << " TStopwatch timer;" << endl;
3078 out << " timer.Start();" << endl << endl;
13f0c00b 3079 // Change temp directory to current one
3080 out << "// Set temporary merging directory to current one" << endl;
3081 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3082 out << "// Set temporary compilation directory to current one" << endl;
3083 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3084 // Reset existing include path
3085 out << "// Reset existing include path and add current directory first in the search" << endl;
3086 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3087 if (!fExecutableCommand.Contains("aliroot")) {
3088 out << "// load base root libraries" << endl;
3089 out << " gSystem->Load(\"libTree\");" << endl;
3090 out << " gSystem->Load(\"libGeom\");" << endl;
3091 out << " gSystem->Load(\"libVMC\");" << endl;
3092 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3093 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3094 }
d5c6455a 3095 if (fAdditionalRootLibs.Length()) {
47a4137d 3096 // in principle libtree /lib geom libvmc etc. can go into this list, too
3097 out << "// Add aditional libraries" << endl;
3098 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3099 TIter next(list);
3100 TObjString *str;
3101 while((str=(TObjString*)next())) {
3102 if (str->GetString().Contains(".so"))
3103 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3104 }
47a4137d 3105 if (list) delete list;
d5c6455a 3106 }
57377eb5 3107 out << "// Load analysis framework libraries" << endl;
652561d5 3108 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3109 if (!fPackages) {
f790bc1b 3110 if (!fExecutableCommand.Contains("aliroot")) {
3111 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3112 out << " gSystem->Load(\"libESD\");" << endl;
3113 out << " gSystem->Load(\"libAOD\");" << endl;
3114 }
0bc15bcd 3115 out << " gSystem->Load(\"libOADB\");" << endl;
4e5c5506 3116 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3117 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3118 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3119 } else {
4e5c5506 3120 TIter next(fPackages);
3121 TObject *obj;
57377eb5 3122 TString pkgname;
57377eb5 3123 while ((obj=next())) {
3124 pkgname = obj->GetName();
4478e6f1 3125 if (pkgname == "STEERBase" ||
3126 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3127 if (pkgname == "ESD" ||
3128 pkgname == "ESD.par") hasESD = kTRUE;
3129 if (pkgname == "AOD" ||
3130 pkgname == "AOD.par") hasAOD = kTRUE;
3131 if (pkgname == "ANALYSIS" ||
3132 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3133 if (pkgname == "ANALYSISalice" ||
3134 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3135 if (pkgname == "CORRFW" ||
3136 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3137 }
3138 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3139 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3140 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3141 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3142 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3143 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3144 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 3145 out << " gSystem->Load(\"libOADB\");" << endl;
57377eb5 3146 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3147 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3148 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3149 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3150 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3151 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3152 out << "// Compile other par packages" << endl;
3153 next.Reset();
fcc9bb6f 3154 while ((obj=next())) {
3155 pkgname = obj->GetName();
4478e6f1 3156 if (pkgname == "STEERBase" ||
3157 pkgname == "STEERBase.par" ||
3158 pkgname == "ESD" ||
3159 pkgname == "ESD.par" ||
3160 pkgname == "AOD" ||
3161 pkgname == "AOD.par" ||
3162 pkgname == "ANALYSIS" ||
3163 pkgname == "ANALYSIS.par" ||
3164 pkgname == "ANALYSISalice" ||
3165 pkgname == "ANALYSISalice.par" ||
3166 pkgname == "CORRFW" ||
3167 pkgname == "CORRFW.par") continue;
5b273635 3168 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3169 }
4e5c5506 3170 }
b3e07543 3171 out << "// include path" << endl;
3172 // Get the include path from the interpreter and remove entries pointing to AliRoot
3173 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3174 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3175 out << " TIter nextpath(listpaths);" << endl;
3176 out << " TObjString *pname;" << endl;
3177 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3178 out << " TString current = pname->GetName();" << endl;
3179 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3180 out << " gSystem->AddIncludePath(current);" << endl;
3181 out << " }" << endl;
3182 out << " if (listpaths) delete listpaths;" << endl;
3183 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3184 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3185 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3186 if (fAdditionalLibs.Length()) {
3187 out << "// Add aditional AliRoot libraries" << endl;
3188 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3189 TIter next(list);
3190 TObjString *str;
3191 while((str=(TObjString*)next())) {
3192 if (str->GetString().Contains(".so"))
3193 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3194 if (str->GetString().Contains(".par"))
3195 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3196 }
3197 if (list) delete list;
3198 }
3199 out << endl;
c57f56b7 3200 out << "// analysis source to be compiled at runtime (if any)" << endl;
3201 if (fAnalysisSource.Length()) {
3202 TObjArray *list = fAnalysisSource.Tokenize(" ");
3203 TIter next(list);
3204 TObjString *str;
3205 while((str=(TObjString*)next())) {
3206 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3207 }
3208 if (list) delete list;
3209 }
3210 out << endl;
b3e07543 3211// out << " printf(\"Currently load libraries:\\n\");" << endl;
3212// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3213 if (fFastReadOption) {
d3339be3 3214 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3215 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3216 out << "// fast xrootd reading enabled" << endl;
3217 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3218 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3219 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3220 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3221 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3222 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3223 }
c57f56b7 3224 out << "// connect to AliEn and make the chain" << endl;
3225 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3226 out << "// read the analysis manager from file" << endl;
f10e8481 3227 TString analysisFile = fExecutable;
3228 analysisFile.ReplaceAll(".sh", ".root");
3229 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3230 out << " if (!file) return;" << endl;
3231 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3232 out << " AliAnalysisManager *mgr = 0;" << endl;
3233 out << " TKey *key;" << endl;
3234 out << " while ((key=(TKey*)nextkey())) {" << endl;
3235 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3236 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3237 out << " };" << endl;
3238 out << " if (!mgr) {" << endl;
205b201f 3239 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3240 out << " return;" << endl;
3241 out << " }" << endl << endl;
3242 out << " mgr->PrintStatus();" << endl;
52b6a92b 3243 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3244 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3245 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3246 } else {
9a8288ff 3247 if (TestBit(AliAnalysisGrid::kTest))
3248 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3249 else
3250 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3251 }
3252 }
242accb2 3253 if (IsUsingTags()) {
3254 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3255 } else {
3256 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3257 }
c57f56b7 3258 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3259 out << " timer.Stop();" << endl;
3260 out << " timer.Print();" << endl;
c57f56b7 3261 out << "}" << endl << endl;
3262 if (IsUsingTags()) {
3263 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3264 out << "{" << endl;
3265 out << "// Create a chain using tags from the xml file." << endl;
3266 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3267 out << " if (!coll) {" << endl;
3268 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3269 out << " return NULL;" << endl;
3270 out << " }" << endl;
3271 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3272 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3273 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3274 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3275 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3276 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3277 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3278 out << " // Check if the cuts configuration file was provided" << endl;
3279 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3280 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3281 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3282 out << " }" << endl;
0df6ccf2 3283 if (fFriendChainName=="") {
3284 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3285 } else {
3286 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3287 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3288 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3289 }
c57f56b7 3290 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3291 out << " chain->ls();" << endl;
3292 out << " return chain;" << endl;
fcc9bb6f 3293 out << "}" << endl << endl;
c57f56b7 3294 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3295 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3296 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3297 msg += " AliLHCTagCuts *lhcCuts,\n";
3298 msg += " AliDetectorTagCuts *detCuts,\n";
3299 msg += " AliEventTagCuts *evCuts)";
23329835 3300 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3301 }
0df6ccf2 3302 }
3303 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3304 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3305 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3306 out << "{" << endl;
3307 out << "// Create a chain using url's from xml file" << endl;
242accb2 3308 out << " TString filename;" << endl;
3309 out << " Int_t run = 0;" << endl;
c85cfc0f 3310 if (IsUseMCchain()) {
3311 out << " TString treename = \"TE\";" << endl;
3312 } else {
3313 out << " TString treename = type;" << endl;
3314 out << " treename.ToLower();" << endl;
3315 out << " treename += \"Tree\";" << endl;
3316 }
e02fee64 3317 out << " printf(\"***************************************\\n\");" << endl;
3318 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3319 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3320 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3321 out << " if (!coll) {" << endl;
3322 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3323 out << " return NULL;" << endl;
3324 out << " }" << endl;
242accb2 3325 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3326 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3327 if(fFriendChainName!="") {
3328 out << " TChain *chainFriend = new TChain(treename);" << endl;
3329 }
c57f56b7 3330 out << " coll->Reset();" << endl;
0df6ccf2 3331 out << " while (coll->Next()) {" << endl;
242accb2 3332 out << " filename = coll->GetTURL("");" << endl;
3333 out << " if (mgr) {" << endl;
3334 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3335 out << " if (nrun && nrun != run) {" << endl;
3336 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3337 out << " mgr->SetRunFromPath(nrun);" << endl;
3338 out << " run = nrun;" << endl;
3339 out << " }" << endl;
3340 out << " }" << endl;
3341 out << " chain->Add(filename);" << endl;
0df6ccf2 3342 if(fFriendChainName!="") {
3343 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3344 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3345 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3346 out << " chainFriend->Add(fileFriend.Data());" << endl;
3347 }
3348 out << " }" << endl;
c57f56b7 3349 out << " if (!chain->GetNtrees()) {" << endl;
3350 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3351 out << " return NULL;" << endl;
3352 out << " }" << endl;
0df6ccf2 3353 if(fFriendChainName!="") {
3354 out << " chain->AddFriend(chainFriend);" << endl;
3355 }
c57f56b7 3356 out << " return chain;" << endl;
fcc9bb6f 3357 out << "}" << endl << endl;
c57f56b7 3358 }
5b273635 3359 if (hasANALYSISalice) {
3360 out <<"//________________________________________________________________________________" << endl;
3361 out << "Bool_t SetupPar(const char *package) {" << endl;
3362 out << "// Compile the package and set it up." << endl;
3363 out << " TString pkgdir = package;" << endl;
3364 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3365 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3366 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3367 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3368 out << " // Check for BUILD.sh and execute" << endl;
3369 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3370 out << " printf(\"*******************************\\n\");" << endl;
3371 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3372 out << " printf(\"*******************************\\n\");" << endl;
3373 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3374 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3375 out << " gSystem->ChangeDirectory(cdir);" << endl;
3376 out << " return kFALSE;" << endl;
3377 out << " }" << endl;
3378 out << " } else {" << endl;
3379 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3380 out << " gSystem->ChangeDirectory(cdir);" << endl;
3381 out << " return kFALSE;" << endl;
3382 out << " }" << endl;
3383 out << " // Check for SETUP.C and execute" << endl;
3384 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3385 out << " printf(\"*******************************\\n\");" << endl;
3386 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3387 out << " printf(\"*******************************\\n\");" << endl;
3388 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3389 out << " } else {" << endl;
3390 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3391 out << " gSystem->ChangeDirectory(cdir);" << endl;
3392 out << " return kFALSE;" << endl;
3393 out << " }" << endl;
3394 out << " // Restore original workdir" << endl;
3395 out << " gSystem->ChangeDirectory(cdir);" << endl;
3396 out << " return kTRUE;" << endl;
3397 out << "}" << endl;
3398 }
c57f56b7 3399 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3400 }
3401 Bool_t copy = kTRUE;
d3b18c4c 3402 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3403 if (copy) {
3404 CdWork();
3405 TString workdir = gGrid->GetHomeDirectory();
3406 workdir += fGridWorkingDir;
3407 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3408 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3409 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3410 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3411 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3412 }
3413 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3414 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3415 }
3416}
3417
0f389141 3418//______________________________________________________________________________
3419void AliAnalysisAlien::WriteMergingMacro()
3420{
3421// Write a macro to merge the outputs per master job.
3422 if (!fMergeViaJDL) return;
3423 if (!fOutputFiles.Length()) {
3424 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3425 return;
3426 }
3427 TString mergingMacro = fExecutable;
3428 mergingMacro.ReplaceAll(".sh","_merge.C");
dc030a24 3429 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
0f389141 3430 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3431 ofstream out;
3432 out.open(mergingMacro.Data(), ios::out);
3433 if (!out.good()) {
3434 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3435 return;
3436 }
5b273635 3437 Bool_t hasSTEERBase = kFALSE;
3438 Bool_t hasESD = kFALSE;
3439 Bool_t hasAOD = kFALSE;
3440 Bool_t hasANALYSIS = kFALSE;
3441 Bool_t hasANALYSISalice = kFALSE;
3442 Bool_t hasCORRFW = kFALSE;
0f389141 3443 TString func = mergingMacro;
3444 TString comment;
3445 func.ReplaceAll(".C", "");
f47d5cb4 3446 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3447 out << "{" << endl;
3448 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3449 out << " TStopwatch timer;" << endl;
3450 out << " timer.Start();" << endl << endl;
b3e07543 3451 // Reset existing include path
3452 out << "// Reset existing include path and add current directory first in the search" << endl;
3453 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3454 if (!fExecutableCommand.Contains("aliroot")) {
3455 out << "// load base root libraries" << endl;
3456 out << " gSystem->Load(\"libTree\");" << endl;
3457 out << " gSystem->Load(\"libGeom\");" << endl;
3458 out << " gSystem->Load(\"libVMC\");" << endl;
3459 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3460 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3461 }
0f389141 3462 if (fAdditionalRootLibs.Length()) {
3463 // in principle libtree /lib geom libvmc etc. can go into this list, too
3464 out << "// Add aditional libraries" << endl;
3465 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3466 TIter next(list);
3467 TObjString *str;
3468 while((str=(TObjString*)next())) {
3469 if (str->GetString().Contains(".so"))
3470 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3471 }
3472 if (list) delete list;
3473 }
0f389141 3474 out << "// Load analysis framework libraries" << endl;
3475 if (!fPackages) {
7c2cd90a 3476 if (!fExecutableCommand.Contains("aliroot")) {
3477 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3478 out << " gSystem->Load(\"libESD\");" << endl;
3479 out << " gSystem->Load(\"libAOD\");" << endl;
3480 }
0bc15bcd 3481 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 3482 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3483 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3484 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3485 } else {
3486 TIter next(fPackages);
3487 TObject *obj;
3488 TString pkgname;
5b273635 3489 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3490 while ((obj=next())) {
3491 pkgname = obj->GetName();
3492 if (pkgname == "STEERBase" ||
3493 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3494 if (pkgname == "ESD" ||
3495 pkgname == "ESD.par") hasESD = kTRUE;
3496 if (pkgname == "AOD" ||
3497 pkgname == "AOD.par") hasAOD = kTRUE;
3498 if (pkgname == "ANALYSIS" ||
3499 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3500 if (pkgname == "ANALYSISalice" ||
3501 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3502 if (pkgname == "CORRFW" ||
3503 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3504 }
5b273635 3505 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3506 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3507 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3508 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3509 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3510 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3511 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0bc15bcd 3512 out << " gSystem->Load(\"libOADB\");" << endl;
0f389141 3513 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3514 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3515 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3516 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3517 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3518 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3519 out << "// Compile other par packages" << endl;
3520 next.Reset();
3521 while ((obj=next())) {
3522 pkgname = obj->GetName();
3523 if (pkgname == "STEERBase" ||
3524 pkgname == "STEERBase.par" ||
3525 pkgname == "ESD" ||
3526 pkgname == "ESD.par" ||
3527 pkgname == "AOD" ||
3528 pkgname == "AOD.par" ||
3529 pkgname == "ANALYSIS" ||
3530 pkgname == "ANALYSIS.par" ||
3531 pkgname == "ANALYSISalice" ||
3532 pkgname == "ANALYSISalice.par" ||
3533 pkgname == "CORRFW" ||
3534 pkgname == "CORRFW.par") continue;
5b273635 3535 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3536 }
3537 }
b3e07543 3538 out << "// include path" << endl;
3539 // Get the include path from the interpreter and remove entries pointing to AliRoot
3540 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3541 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3542 out << " TIter nextpath(listpaths);" << endl;
3543 out << " TObjString *pname;" << endl;
3544 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3545 out << " TString current = pname->GetName();" << endl;
3546 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3547 out << " gSystem->AddIncludePath(current);" << endl;
3548 out << " }" << endl;
3549 out << " if (listpaths) delete listpaths;" << endl;
3550 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3551 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3552 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3553 if (fAdditionalLibs.Length()) {
3554 out << "// Add aditional AliRoot libraries" << endl;
3555 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3556 TIter next(list);
3557 TObjString *str;
3558 while((str=(TObjString*)next())) {
3559 if (str->GetString().Contains(".so"))
3560 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3561 }
3562 if (list) delete list;
3563 }
3564 out << endl;
3565 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3566 if (fAnalysisSource.Length()) {
3567 TObjArray *list = fAnalysisSource.Tokenize(" ");
3568 TIter next(list);
3569 TObjString *str;
3570 while((str=(TObjString*)next())) {
3571 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3572 }
3573 if (list) delete list;
3574 }
149d288c 3575 out << endl;
3576
0f389141 3577 if (fFastReadOption) {
3578 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3579 out << "// fast xrootd reading enabled" << endl;
3580 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3581 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3582 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3583 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3584 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3585 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3586 }
3587 // Change temp directory to current one
3588 out << "// Set temporary merging directory to current one" << endl;
3589 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3590 out << "// Set temporary compilation directory to current one" << endl;
3591 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 3592 out << "// Connect to AliEn" << endl;
3593 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3594 out << " TString outputDir = dir;" << endl;
f790bc1b 3595 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3596 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3597 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3598 out << " TIter *iter = new TIter(list);" << endl;
3599 out << " TObjString *str;" << endl;
a2f5fc01 3600 out << " TString outputFile;" << endl;
0f389141 3601 out << " Bool_t merged = kTRUE;" << endl;
3602 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3603 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3604 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3605 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3606 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3607 out << " // Skip already merged outputs" << endl;
a2f5fc01 3608 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3609 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3610 out << " continue;" << endl;
3611 out << " }" << endl;
a2f5fc01 3612 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3613 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3614 out << " if (!merged) {" << endl;
a2f5fc01 3615 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3616 out << " return;" << endl;
0f389141 3617 out << " }" << endl;
3618 out << " }" << endl;
7c2cd90a 3619 out << " // all outputs merged, validate" << endl;
3620 out << " ofstream out;" << endl;
3621 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3622 out << " out.close();" << endl;
3623 out << " // read the analysis manager from file" << endl;
f866cba5 3624 TString analysisFile = fExecutable;
3625 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3626 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3627 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3628 out << " if (!file) return;" << endl;
f866cba5 3629 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3630 out << " AliAnalysisManager *mgr = 0;" << endl;
3631 out << " TKey *key;" << endl;
3632 out << " while ((key=(TKey*)nextkey())) {" << endl;
3633 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3634 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3635 out << " };" << endl;
3636 out << " if (!mgr) {" << endl;
3637 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3638 out << " return;" << endl;
3639 out << " }" << endl << endl;
5c4250fc 3640 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3641 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3642 out << " mgr->PrintStatus();" << endl;
3643 if (AliAnalysisManager::GetAnalysisManager()) {
3644 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3645 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3646 } else {
9a8288ff 3647 if (TestBit(AliAnalysisGrid::kTest))
3648 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3649 else
3650 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3651 }
3652 }
9a8288ff 3653 out << " TTree *tree = NULL;" << endl;
3654 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3655 out << "}" << endl << endl;
5b273635 3656 if (hasANALYSISalice) {
3657 out <<"//________________________________________________________________________________" << endl;
3658 out << "Bool_t SetupPar(const char *package) {" << endl;
3659 out << "// Compile the package and set it up." << endl;
3660 out << " TString pkgdir = package;" << endl;
3661 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3662 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3663 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3664 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3665 out << " // Check for BUILD.sh and execute" << endl;
3666 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3667 out << " printf(\"*******************************\\n\");" << endl;
3668 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3669 out << " printf(\"*******************************\\n\");" << endl;
3670 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3671 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3672 out << " gSystem->ChangeDirectory(cdir);" << endl;
3673 out << " return kFALSE;" << endl;
3674 out << " }" << endl;
3675 out << " } else {" << endl;
3676 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3677 out << " gSystem->ChangeDirectory(cdir);" << endl;
3678 out << " return kFALSE;" << endl;
3679 out << " }" << endl;
3680 out << " // Check for SETUP.C and execute" << endl;
3681 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3682 out << " printf(\"*******************************\\n\");" << endl;
3683 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3684 out << " printf(\"*******************************\\n\");" << endl;
3685 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3686 out << " } else {" << endl;
3687 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3688 out << " gSystem->ChangeDirectory(cdir);" << endl;
3689 out << " return kFALSE;" << endl;
3690 out << " }" << endl;
3691 out << " // Restore original workdir" << endl;
3692 out << " gSystem->ChangeDirectory(cdir);" << endl;
3693 out << " return kTRUE;" << endl;
3694 out << "}" << endl;
3695 }
0f389141 3696 }
3697 Bool_t copy = kTRUE;
d3b18c4c 3698 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3699 if (copy) {
3700 CdWork();
3701 TString workdir = gGrid->GetHomeDirectory();
3702 workdir += fGridWorkingDir;
3703 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3704 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3705 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3706 }
3707}
3708
3709//______________________________________________________________________________
3710Bool_t AliAnalysisAlien::SetupPar(const char *package)
3711{
205b201f 3712// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3713// Note that for loading the compiled library. The current directory should have precedence in
3714// LD_LIBRARY_PATH
3715 TString pkgdir = package;
3716 pkgdir.ReplaceAll(".par","");
b50a41c3 3717 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3718 TString cdir = gSystem->WorkingDirectory();
3719 gSystem->ChangeDirectory(pkgdir);
3720 // Check for BUILD.sh and execute
3721 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3722 printf("**************************************************\n");
3723 printf("*** Building PAR archive %s\n", package);
3724 printf("**************************************************\n");
3725 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3726 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3727 gSystem->ChangeDirectory(cdir);
3728 return kFALSE;
3729 }
3730 } else {
3731 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3732 gSystem->ChangeDirectory(cdir);
3733 return kFALSE;
3734 }
3735 // Check for SETUP.C and execute
3736 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3737 printf("**************************************************\n");
3738 printf("*** Setup PAR archive %s\n", package);
3739 printf("**************************************************\n");
3740 gROOT->Macro("PROOF-INF/SETUP.C");
3741 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3742 } else {
3743 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3744 gSystem->ChangeDirectory(cdir);
3745 return kFALSE;
3746 }
3747 // Restore original workdir
3748 gSystem->ChangeDirectory(cdir);
3749 return kTRUE;
3750}
3751
c57f56b7 3752//______________________________________________________________________________
3753void AliAnalysisAlien::WriteExecutable()
3754{
3755// Generate the alien executable script.
3756 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3757 ofstream out;
3758 out.open(fExecutable.Data(), ios::out);
3759 if (out.bad()) {
5513444a 3760 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3761 return;
3762 }
3763 out << "#!/bin/bash" << endl;
2a4d5166 3764 // Make sure we can properly compile par files
3765 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3766 out << "echo \"=========================================\"" << endl;
3767 out << "echo \"############## PATH : ##############\"" << endl;
3768 out << "echo $PATH" << endl;
3769 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3770 out << "echo $LD_LIBRARY_PATH" << endl;
3771 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3772 out << "echo $ROOTSYS" << endl;
3773 out << "echo \"############## which root : ##############\"" << endl;
3774 out << "which root" << endl;
3775 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3776 out << "echo $ALICE_ROOT" << endl;
3777 out << "echo \"############## which aliroot : ##############\"" << endl;
3778 out << "which aliroot" << endl;
9c5ddadc 3779 out << "echo \"############## system limits : ##############\"" << endl;
3780 out << "ulimit -a" << endl;
3781 out << "echo \"############## memory : ##############\"" << endl;
3782 out << "free -m" << endl;
c57f56b7 3783 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3784 out << fExecutableCommand << " ";
631c0b05 3785 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3786 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3787 out << "echo \"############## memory after: ##############\"" << endl;
3788 out << "free -m" << endl;
c57f56b7 3789 }
3790 Bool_t copy = kTRUE;
d3b18c4c 3791 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3792 if (copy) {
3793 CdWork();
3794 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3795 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3796 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3797 workdir += fGridWorkingDir;
3798 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3799 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3800 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3801 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3802 }
3803}
3804
0f389141 3805//______________________________________________________________________________
3806void AliAnalysisAlien::WriteMergeExecutable()
3807{
3808// Generate the alien executable script for the merging job.
3809 if (!fMergeViaJDL) return;
3810 TString mergeExec = fExecutable;
3811 mergeExec.ReplaceAll(".sh", "_merge.sh");
3812 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3813 ofstream out;
3814 out.open(mergeExec.Data(), ios::out);
3815 if (out.bad()) {
3816 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3817 return;
3818 }
3819 out << "#!/bin/bash" << endl;
2a4d5166 3820 // Make sure we can properly compile par files
3821 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3822 out << "echo \"=========================================\"" << endl;
3823 out << "echo \"############## PATH : ##############\"" << endl;
3824 out << "echo $PATH" << endl;
3825 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3826 out << "echo $LD_LIBRARY_PATH" << endl;
3827 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3828 out << "echo $ROOTSYS" << endl;
3829 out << "echo \"############## which root : ##############\"" << endl;
3830 out << "which root" << endl;
3831 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3832 out << "echo $ALICE_ROOT" << endl;
3833 out << "echo \"############## which aliroot : ##############\"" << endl;
3834 out << "which aliroot" << endl;
3835 out << "echo \"############## system limits : ##############\"" << endl;
3836 out << "ulimit -a" << endl;
3837 out << "echo \"############## memory : ##############\"" << endl;
3838 out << "free -m" << endl;
3839 out << "echo \"=========================================\"" << endl << endl;
0f389141 3840 TString mergeMacro = fExecutable;
3841 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3842 if (IsOneStageMerging())
3843 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3844 else
f47d5cb4 3845 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3846 out << fExecutableCommand << " " << "$ARG" << endl;
3847 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3848 out << "echo \"############## memory after: ##############\"" << endl;
3849 out << "free -m" << endl;
0f389141 3850 }
3851 Bool_t copy = kTRUE;
d3b18c4c 3852 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3853 if (copy) {
3854 CdWork();
3855 TString workdir = gGrid->GetHomeDirectory();
3856 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3857 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3858 workdir += fGridWorkingDir;
3859 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3860 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3861 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3862 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3863 }
3864}
3865
c57f56b7 3866//______________________________________________________________________________
5513444a 3867void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3868{
3869// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3870// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3871// Next lines: full_path_to_dataset XXX (XXX is a string)
3872// To submit, one has to: submit jdl XXX for all lines
3873 ofstream out;
3874 out.open(filename, ios::out);
3875 if (out.bad()) {
3876 Error("WriteProductionFile", "Bad file name: %s", filename);
3877 return;
3878 }
d3b18c4c 3879 TString workdir;
3880 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3881 workdir = gGrid->GetHomeDirectory();
5513444a 3882 workdir += fGridWorkingDir;
f5e8c702 3883 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3884 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3885 out << locjdl << " " << njobspermaster << endl;
5513444a 3886 Int_t nmasterjobs = fInputFiles->GetEntries();
3887 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3888 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3889 runOutDir.ReplaceAll(".xml", "");
3890 if (fOutputToRunNo)
3891 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3892 else
3893 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3894 }
d3b18c4c 3895 if (gGrid) {
3896 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3897 if (FileExists(filename)) gGrid->Rm(filename);
3898 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3899 }
5513444a 3900}
3901
3902//______________________________________________________________________________
0f389141 3903void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3904{
3905// Generate the alien validation script.
3906 // Generate the validation script
3907 TObjString *os;
d3b18c4c 3908 if (fValidationScript.IsNull()) {
3909 fValidationScript = fExecutable;
3910 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3911 }
3912 TString validationScript = fValidationScript;
3913 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3914 if (!Connect()) {
3915 Error("WriteValidationScript", "Alien connection required");
3916 return;
3917 }
d3b18c4c 3918 if (!fTerminateFiles.IsNull()) {
3919 fTerminateFiles.Strip();
3920 fTerminateFiles.ReplaceAll(" ",",");
3921 }
a2f5fc01 3922 TString outStream = "";
3923 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3924 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3925 ofstream out;
0d5d317c 3926 out.open(validationScript, ios::out);
c57f56b7 3927 out << "#!/bin/bash" << endl;
3928 out << "##################################################" << endl;
3929 out << "validateout=`dirname $0`" << endl;
3930 out << "validatetime=`date`" << endl;
3931 out << "validated=\"0\";" << endl;
3932 out << "error=0" << endl;
3933 out << "if [ -z $validateout ]" << endl;
3934 out << "then" << endl;
3935 out << " validateout=\".\"" << endl;
3936 out << "fi" << endl << endl;
3937 out << "cd $validateout;" << endl;
3938 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3939 out << "echo \"*******************************************************\"" << outStream << endl;
3940 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3941 out << "" << endl;
a2f5fc01 3942 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3943 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3944 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3945 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3946 out << "ls -la ./" << outStream << endl;
3947 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3948 out << "##################################################" << endl;
ebec370a 3949 out << "" << endl;
3950
3951 out << "if [ ! -f stderr ] ; then" << endl;
3952 out << " error=1" << endl;
a2f5fc01 3953 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3954 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3955 out << "fi" << endl;
3956
b34c9f51 3957 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3958 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3959 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3960 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3961 out << "" << endl;
3962
ebec370a 3963 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3964 out << " error=1" << endl;
a2f5fc01 3965 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3966 out << " echo \"$parArch\" " << outStream << endl;
3967 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3968 out << "fi" << endl;
3969
3970 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3971 out << " error=1" << endl;
a2f5fc01 3972 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3973 out << " echo \"$segViol\" " << outStream << endl;
3974 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3975 out << "fi" << endl;
3976
3977 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3978 out << " error=1" << endl;
a2f5fc01 3979 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3980 out << " echo \"$segFault\" " << outStream << endl;
3981 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3982 out << "fi" << endl;
3983
b34c9f51 3984 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3985 out << " error=1" << endl;
a2f5fc01 3986 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3987 out << " echo \"$glibcErr\" " << outStream << endl;
3988 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3989 out << "fi" << endl;
3990
ebec370a 3991 // Part dedicated to the specific analyses running into the train
3992
d3b18c4c 3993 TString outputFiles = fOutputFiles;
3994 if (merge && !fTerminateFiles.IsNull()) {
3995 outputFiles += ",";
3996 outputFiles += fTerminateFiles;
3997 }
3998 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3999 TIter next1(arr);
a2f5fc01 4000 TString outputFile;
bbd88db8 4001 while (!merge && (os=(TObjString*)next1())) {
4002 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 4003 outputFile = os->GetString();
4004 Int_t index = outputFile.Index("@");
4005 if (index > 0) outputFile.Remove(index);
bbd88db8 4006 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 4007 if (outputFile.Contains("*")) continue;
4008 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 4009 out << " error=1" << endl;
7c2cd90a 4010 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
4011 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 4012 out << "fi" << endl;
4013 }
4014 delete arr;
7c2cd90a 4015 out << "if ! [ -f outputs_valid ] ; then" << endl;
4016 out << " error=1" << endl;
4017 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4018 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4019 out << "fi" << endl;
923e2ca5 4020
c57f56b7 4021 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4022 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4023 if (!IsKeepLogs()) {
4024 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4025 outStream = "";
149d288c 4026 out << " rm -f std*" << endl;
4027 }
c57f56b7 4028 out << "fi" << endl;
4029
a2f5fc01 4030 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4031 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4032 out << "cd -" << endl;
4033 out << "exit $error" << endl;
4034 }
4035 Bool_t copy = kTRUE;
d3b18c4c 4036 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4037 if (copy) {
4038 CdWork();
4039 TString workdir = gGrid->GetHomeDirectory();
4040 workdir += fGridWorkingDir;
d3b18c4c 4041 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4042 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4043 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4044 }
4045}