]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Coverity fixes
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
65eb22f8 23#include "AliAnalysisAlien.h"
24
c57f56b7 25#include "Riostream.h"
0f389141 26#include "TEnv.h"
7c2cd90a 27#include "TBits.h"
0f389141 28#include "TError.h"
c57f56b7 29#include "TROOT.h"
30#include "TSystem.h"
31#include "TFile.h"
830acc4c 32#include "TFileCollection.h"
3bdcb562 33#include "TChain.h"
c57f56b7 34#include "TObjString.h"
35#include "TObjArray.h"
36#include "TGrid.h"
37#include "TGridResult.h"
38#include "TGridCollection.h"
39#include "TGridJDL.h"
d2a409b2 40#include "TGridJobStatusList.h"
41#include "TGridJobStatus.h"
c57f56b7 42#include "TFileMerger.h"
43#include "AliAnalysisManager.h"
bb885a9e 44#include "AliVEventHandler.h"
45#include "AliAnalysisDataContainer.h"
65eb22f8 46#include "AliMultiInputEventHandler.h"
c57f56b7 47
48ClassImp(AliAnalysisAlien)
49
50//______________________________________________________________________________
51AliAnalysisAlien::AliAnalysisAlien()
52 :AliAnalysisGrid(),
53 fGridJDL(NULL),
0f389141 54 fMergingJDL(NULL),
c57f56b7 55 fPrice(0),
56 fTTL(0),
57 fSplitMaxInputFileNumber(0),
58 fMaxInitFailed(0),
59 fMasterResubmitThreshold(0),
bb885a9e 60 fNtestFiles(0),
319593fb 61 fNrunsPerMaster(0),
16a4353c 62 fMaxMergeFiles(0),
b3e07543 63 fMaxMergeStages(0),
d2a409b2 64 fNsubmitted(0),
a3e84053 65 fProductionMode(0),
cd11251e 66 fOutputToRunNo(0),
0f389141 67 fMergeViaJDL(0),
68 fFastReadOption(0),
e1c22e21 69 fOverwriteMode(1),
149d288c 70 fNreplicas(2),
3bdcb562 71 fNproofWorkers(0),
27734f0e 72 fNproofWorkersPerSlave(0),
3bdcb562 73 fProofReset(0),
c57f56b7 74 fRunNumbers(),
75 fExecutable(),
0a1c1f7f 76 fExecutableCommand(),
c57f56b7 77 fArguments(),
631c0b05 78 fExecutableArgs(),
c57f56b7 79 fAnalysisMacro(),
80 fAnalysisSource(),
d3b18c4c 81 fValidationScript(),
d5c6455a 82 fAdditionalRootLibs(),
c57f56b7 83 fAdditionalLibs(),
84 fSplitMode(),
85 fAPIVersion(),
86 fROOTVersion(),
87 fAliROOTVersion(),
648174cf 88 fExternalPackages(),
c57f56b7 89 fUser(),
90 fGridWorkingDir(),
91 fGridDataDir(),
92 fDataPattern(),
93 fGridOutputDir(),
94 fOutputArchive(),
95 fOutputFiles(),
96 fInputFormat(),
e7c71df0 97 fDatasetName(),
c57f56b7 98 fJDLName(),
d3b18c4c 99 fTerminateFiles(),
bb885a9e 100 fMergeExcludes(),
f965131e 101 fIncludePath(),
bb885a9e 102 fCloseSE(),
0df6ccf2 103 fFriendChainName(),
c6cb3634 104 fJobTag(),
648174cf 105 fOutputSingle(),
5fce53f4 106 fRunPrefix(),
3bdcb562 107 fProofCluster(),
108 fProofDataSet(),
109 fFileForTestMode(),
110 fRootVersionForProof(),
111 fAliRootMode(),
f47d5cb4 112 fMergeDirName(),
4e5c5506 113 fInputFiles(0),
114 fPackages(0)
c57f56b7 115{
116// Dummy ctor.
117 SetDefaults();
118}
119
120//______________________________________________________________________________
121AliAnalysisAlien::AliAnalysisAlien(const char *name)
122 :AliAnalysisGrid(name),
123 fGridJDL(NULL),
0f389141 124 fMergingJDL(NULL),
c57f56b7 125 fPrice(0),
126 fTTL(0),
127 fSplitMaxInputFileNumber(0),
128 fMaxInitFailed(0),
129 fMasterResubmitThreshold(0),
bb885a9e 130 fNtestFiles(0),
319593fb 131 fNrunsPerMaster(0),
16a4353c 132 fMaxMergeFiles(0),
b3e07543 133 fMaxMergeStages(0),
d2a409b2 134 fNsubmitted(0),
a3e84053 135 fProductionMode(0),
cd11251e 136 fOutputToRunNo(0),
0f389141 137 fMergeViaJDL(0),
138 fFastReadOption(0),
e1c22e21 139 fOverwriteMode(1),
149d288c 140 fNreplicas(2),
3bdcb562 141 fNproofWorkers(0),
27734f0e 142 fNproofWorkersPerSlave(0),
3bdcb562 143 fProofReset(0),
c57f56b7 144 fRunNumbers(),
145 fExecutable(),
0a1c1f7f 146 fExecutableCommand(),
c57f56b7 147 fArguments(),
631c0b05 148 fExecutableArgs(),
c57f56b7 149 fAnalysisMacro(),
150 fAnalysisSource(),
d3b18c4c 151 fValidationScript(),
d5c6455a 152 fAdditionalRootLibs(),
c57f56b7 153 fAdditionalLibs(),
154 fSplitMode(),
155 fAPIVersion(),
156 fROOTVersion(),
157 fAliROOTVersion(),
648174cf 158 fExternalPackages(),
c57f56b7 159 fUser(),
160 fGridWorkingDir(),
161 fGridDataDir(),
162 fDataPattern(),
163 fGridOutputDir(),
164 fOutputArchive(),
165 fOutputFiles(),
166 fInputFormat(),
e7c71df0 167 fDatasetName(),
c57f56b7 168 fJDLName(),
d3b18c4c 169 fTerminateFiles(),
bb885a9e 170 fMergeExcludes(),
f965131e 171 fIncludePath(),
bb885a9e 172 fCloseSE(),
0df6ccf2 173 fFriendChainName(),
c6cb3634 174 fJobTag(),
648174cf 175 fOutputSingle(),
5fce53f4 176 fRunPrefix(),
3bdcb562 177 fProofCluster(),
178 fProofDataSet(),
179 fFileForTestMode(),
180 fRootVersionForProof(),
181 fAliRootMode(),
f47d5cb4 182 fMergeDirName(),
4e5c5506 183 fInputFiles(0),
184 fPackages(0)
c57f56b7 185{
186// Default ctor.
187 SetDefaults();
188}
189
190//______________________________________________________________________________
191AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
192 :AliAnalysisGrid(other),
193 fGridJDL(NULL),
0f389141 194 fMergingJDL(NULL),
c57f56b7 195 fPrice(other.fPrice),
196 fTTL(other.fTTL),
197 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
198 fMaxInitFailed(other.fMaxInitFailed),
199 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 200 fNtestFiles(other.fNtestFiles),
319593fb 201 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 202 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 203 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 204 fNsubmitted(other.fNsubmitted),
a3e84053 205 fProductionMode(other.fProductionMode),
cd11251e 206 fOutputToRunNo(other.fOutputToRunNo),
0f389141 207 fMergeViaJDL(other.fMergeViaJDL),
208 fFastReadOption(other.fFastReadOption),
209 fOverwriteMode(other.fOverwriteMode),
149d288c 210 fNreplicas(other.fNreplicas),
3bdcb562 211 fNproofWorkers(other.fNproofWorkers),
27734f0e 212 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 213 fProofReset(other.fProofReset),
c57f56b7 214 fRunNumbers(other.fRunNumbers),
215 fExecutable(other.fExecutable),
0a1c1f7f 216 fExecutableCommand(other.fExecutableCommand),
c57f56b7 217 fArguments(other.fArguments),
631c0b05 218 fExecutableArgs(other.fExecutableArgs),
c57f56b7 219 fAnalysisMacro(other.fAnalysisMacro),
220 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 221 fValidationScript(other.fValidationScript),
d5c6455a 222 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 223 fAdditionalLibs(other.fAdditionalLibs),
224 fSplitMode(other.fSplitMode),
225 fAPIVersion(other.fAPIVersion),
226 fROOTVersion(other.fROOTVersion),
227 fAliROOTVersion(other.fAliROOTVersion),
648174cf 228 fExternalPackages(other.fExternalPackages),
c57f56b7 229 fUser(other.fUser),
230 fGridWorkingDir(other.fGridWorkingDir),
231 fGridDataDir(other.fGridDataDir),
232 fDataPattern(other.fDataPattern),
233 fGridOutputDir(other.fGridOutputDir),
234 fOutputArchive(other.fOutputArchive),
235 fOutputFiles(other.fOutputFiles),
236 fInputFormat(other.fInputFormat),
e7c71df0 237 fDatasetName(other.fDatasetName),
c57f56b7 238 fJDLName(other.fJDLName),
d3b18c4c 239 fTerminateFiles(other.fTerminateFiles),
bb885a9e 240 fMergeExcludes(other.fMergeExcludes),
f965131e 241 fIncludePath(other.fIncludePath),
bb885a9e 242 fCloseSE(other.fCloseSE),
0df6ccf2 243 fFriendChainName(other.fFriendChainName),
c6cb3634 244 fJobTag(other.fJobTag),
648174cf 245 fOutputSingle(other.fOutputSingle),
5fce53f4 246 fRunPrefix(other.fRunPrefix),
3bdcb562 247 fProofCluster(other.fProofCluster),
248 fProofDataSet(other.fProofDataSet),
249 fFileForTestMode(other.fFileForTestMode),
250 fRootVersionForProof(other.fRootVersionForProof),
251 fAliRootMode(other.fAliRootMode),
f47d5cb4 252 fMergeDirName(other.fMergeDirName),
4e5c5506 253 fInputFiles(0),
254 fPackages(0)
c57f56b7 255{
256// Copy ctor.
257 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 258 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 259 fRunRange[0] = other.fRunRange[0];
260 fRunRange[1] = other.fRunRange[1];
c57f56b7 261 if (other.fInputFiles) {
262 fInputFiles = new TObjArray();
263 TIter next(other.fInputFiles);
264 TObject *obj;
265 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
266 fInputFiles->SetOwner();
267 }
4e5c5506 268 if (other.fPackages) {
269 fPackages = new TObjArray();
270 TIter next(other.fPackages);
271 TObject *obj;
272 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
273 fPackages->SetOwner();
274 }
c57f56b7 275}
276
277//______________________________________________________________________________
278AliAnalysisAlien::~AliAnalysisAlien()
279{
280// Destructor.
281 if (fGridJDL) delete fGridJDL;
0f389141 282 if (fMergingJDL) delete fMergingJDL;
c57f56b7 283 if (fInputFiles) delete fInputFiles;
4e5c5506 284 if (fPackages) delete fPackages;
c57f56b7 285}
286
287//______________________________________________________________________________
288AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
289{
290// Assignment.
291 if (this != &other) {
292 AliAnalysisGrid::operator=(other);
293 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 294 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 295 fPrice = other.fPrice;
296 fTTL = other.fTTL;
297 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
298 fMaxInitFailed = other.fMaxInitFailed;
299 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 300 fNtestFiles = other.fNtestFiles;
a3e84053 301 fNrunsPerMaster = other.fNrunsPerMaster;
302 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 303 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 304 fNsubmitted = other.fNsubmitted;
305 fProductionMode = other.fProductionMode;
cd11251e 306 fOutputToRunNo = other.fOutputToRunNo;
0f389141 307 fMergeViaJDL = other.fMergeViaJDL;
308 fFastReadOption = other.fFastReadOption;
309 fOverwriteMode = other.fOverwriteMode;
149d288c 310 fNreplicas = other.fNreplicas;
3bdcb562 311 fNproofWorkers = other.fNproofWorkers;
27734f0e 312 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 313 fProofReset = other.fProofReset;
c57f56b7 314 fRunNumbers = other.fRunNumbers;
315 fExecutable = other.fExecutable;
0a1c1f7f 316 fExecutableCommand = other.fExecutableCommand;
c57f56b7 317 fArguments = other.fArguments;
631c0b05 318 fExecutableArgs = other.fExecutableArgs;
c57f56b7 319 fAnalysisMacro = other.fAnalysisMacro;
320 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 321 fValidationScript = other.fValidationScript;
d5c6455a 322 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 323 fAdditionalLibs = other.fAdditionalLibs;
324 fSplitMode = other.fSplitMode;
325 fAPIVersion = other.fAPIVersion;
326 fROOTVersion = other.fROOTVersion;
327 fAliROOTVersion = other.fAliROOTVersion;
648174cf 328 fExternalPackages = other.fExternalPackages;
c57f56b7 329 fUser = other.fUser;
330 fGridWorkingDir = other.fGridWorkingDir;
331 fGridDataDir = other.fGridDataDir;
332 fDataPattern = other.fDataPattern;
333 fGridOutputDir = other.fGridOutputDir;
334 fOutputArchive = other.fOutputArchive;
335 fOutputFiles = other.fOutputFiles;
336 fInputFormat = other.fInputFormat;
e7c71df0 337 fDatasetName = other.fDatasetName;
c57f56b7 338 fJDLName = other.fJDLName;
d3b18c4c 339 fTerminateFiles = other.fTerminateFiles;
bb885a9e 340 fMergeExcludes = other.fMergeExcludes;
f965131e 341 fIncludePath = other.fIncludePath;
bb885a9e 342 fCloseSE = other.fCloseSE;
0df6ccf2 343 fFriendChainName = other.fFriendChainName;
c6cb3634 344 fJobTag = other.fJobTag;
648174cf 345 fOutputSingle = other.fOutputSingle;
5fce53f4 346 fRunPrefix = other.fRunPrefix;
3bdcb562 347 fProofCluster = other.fProofCluster;
348 fProofDataSet = other.fProofDataSet;
349 fFileForTestMode = other.fFileForTestMode;
350 fRootVersionForProof = other.fRootVersionForProof;
351 fAliRootMode = other.fAliRootMode;
f47d5cb4 352 fMergeDirName = other.fMergeDirName;
c57f56b7 353 if (other.fInputFiles) {
354 fInputFiles = new TObjArray();
355 TIter next(other.fInputFiles);
356 TObject *obj;
357 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
358 fInputFiles->SetOwner();
359 }
4e5c5506 360 if (other.fPackages) {
361 fPackages = new TObjArray();
362 TIter next(other.fPackages);
363 TObject *obj;
364 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
365 fPackages->SetOwner();
366 }
c57f56b7 367 }
368 return *this;
369}
370
74013535 371//______________________________________________________________________________
372void AliAnalysisAlien::SetRunPrefix(const char *prefix)
373{
374// Set the run number format. Can be a prefix or a format like "%09d"
375 fRunPrefix = prefix;
376 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
377}
378
f965131e 379//______________________________________________________________________________
380void AliAnalysisAlien::AddIncludePath(const char *path)
381{
382// Add include path in the remote analysis macro.
383 TString p(path);
384 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
385 else fIncludePath += Form("-I%s ", path);
386}
387
c57f56b7 388//______________________________________________________________________________
389void AliAnalysisAlien::AddRunNumber(Int_t run)
390{
391// Add a run number to the list of runs to be processed.
392 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 393 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 394}
395
98690895 396//______________________________________________________________________________
397void AliAnalysisAlien::AddRunList(const char* runList)
398{
399// Add several runs into the list of runs; they are expected to be separated by a blank character.
400 TString sList = runList;
401 TObjArray *list = sList.Tokenize(" ");
402 Int_t n = list->GetEntries();
403 for (Int_t i = 0; i < n; i++) {
404 TObjString *os = (TObjString*)list->At(i);
405 AddRunNumber(os->GetString().Atoi());
406 }
407 delete list;
408}
409
ee75cfc3 410//______________________________________________________________________________
411void AliAnalysisAlien::AddRunNumber(const char* run)
412{
413// Add a run number to the list of runs to be processed.
414 if (fRunNumbers.Length()) fRunNumbers += " ";
415 fRunNumbers += run;
416}
417
c57f56b7 418//______________________________________________________________________________
419void AliAnalysisAlien::AddDataFile(const char *lfn)
420{
421// Adds a data file to the input to be analysed. The file should be a valid LFN
422// or point to an existing file in the alien workdir.
423 if (!fInputFiles) fInputFiles = new TObjArray();
424 fInputFiles->Add(new TObjString(lfn));
425}
648174cf 426
427//______________________________________________________________________________
428void AliAnalysisAlien::AddExternalPackage(const char *package)
429{
430// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
431 if (fExternalPackages) fExternalPackages += " ";
432 fExternalPackages += package;
433}
434
c57f56b7 435//______________________________________________________________________________
436Bool_t AliAnalysisAlien::Connect()
437{
438// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
439 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 440 if (fProductionMode) return kTRUE;
c57f56b7 441 if (!gGrid) {
442 Info("Connect", "Trying to connect to AliEn ...");
443 TGrid::Connect("alien://");
444 }
445 if (!gGrid || !gGrid->IsConnected()) {
446 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
447 return kFALSE;
448 }
449 fUser = gGrid->GetUser();
450 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
451 return kTRUE;
452}
453
454//______________________________________________________________________________
455void AliAnalysisAlien::CdWork()
456{
457// Check validity of alien workspace. Create directory if possible.
458 if (!Connect()) {
459 Error("CdWork", "Alien connection required");
460 return;
461 }
462 TString homedir = gGrid->GetHomeDirectory();
463 TString workdir = homedir + fGridWorkingDir;
923e2ca5 464 if (DirectoryExists(workdir)) {
465 gGrid->Cd(workdir);
466 return;
467 }
468 // Work directory not existing - create it
469 gGrid->Cd(homedir);
b93f8109 470 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 471 gGrid->Cd(fGridWorkingDir);
d3b18c4c 472 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 473 } else {
d3b18c4c 474 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 475 workdir.Data(), homedir.Data());
476 fGridWorkingDir = "";
477 }
c57f56b7 478}
479
348be253 480//______________________________________________________________________________
481Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
482{
483// Check if file copying is possible.
d3b18c4c 484 if (fProductionMode) return kTRUE;
348be253 485 if (!Connect()) {
486 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
487 return kFALSE;
488 }
d3339be3 489 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
490 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 491 // Check if alien_CLOSE_SE is defined
492 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
493 if (!closeSE.IsNull()) {
494 Info("CheckFileCopy", "Your current close storage is pointing to: \
495 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
496 } else {
497 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
498 }
499 // Check if grid directory exists.
500 if (!DirectoryExists(alienpath)) {
501 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
502 return kFALSE;
503 }
d3339be3 504 TFile f("plugin_test_copy", "RECREATE");
348be253 505 // User may not have write permissions to current directory
506 if (f.IsZombie()) {
507 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
508 gSystem->WorkingDirectory());
509 return kFALSE;
510 }
511 f.Close();
d3339be3 512 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
513 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
514 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 515 \n# 1. Make sure you have write permissions there. If this is the case: \
516 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
517 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
518 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
519 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
520 gSystem->Unlink(f.GetName());
521 return kFALSE;
522 }
523 gSystem->Unlink(f.GetName());
d3339be3 524 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
525 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 526 return kTRUE;
527}
528
c57f56b7 529//______________________________________________________________________________
530Bool_t AliAnalysisAlien::CheckInputData()
531{
532// Check validity of input data. If necessary, create xml files.
d3b18c4c 533 if (fProductionMode) return kTRUE;
a8739e8a 534 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 535 if (!fGridDataDir.Length()) {
536 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
537 return kFALSE;
538 }
9f020952 539 if (fMergeViaJDL) {
540 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
541 return kFALSE;
542 }
d2a409b2 543 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 544 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
545 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 546 return kTRUE;
c57f56b7 547 }
548 // Process declared files
a2f5fc01 549 Bool_t isCollection = kFALSE;
550 Bool_t isXml = kFALSE;
551 Bool_t useTags = kFALSE;
c57f56b7 552 Bool_t checked = kFALSE;
d3b18c4c 553 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 554 TString file;
555 TString workdir = gGrid->GetHomeDirectory();
556 workdir += fGridWorkingDir;
557 if (fInputFiles) {
558 TObjString *objstr;
559 TIter next(fInputFiles);
560 while ((objstr=(TObjString*)next())) {
561 file = workdir;
562 file += "/";
563 file += objstr->GetString();
564 // Store full lfn path
565 if (FileExists(file)) objstr->SetString(file);
566 else {
567 file = objstr->GetName();
568 if (!FileExists(objstr->GetName())) {
569 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
570 objstr->GetName(), workdir.Data());
571 return kFALSE;
572 }
573 }
574 Bool_t iscoll, isxml, usetags;
575 CheckDataType(file, iscoll, isxml, usetags);
576 if (!checked) {
577 checked = kTRUE;
a2f5fc01 578 isCollection = iscoll;
579 isXml = isxml;
580 useTags = usetags;
581 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 582 } else {
a2f5fc01 583 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 584 Error("CheckInputData", "Some conflict was found in the types of inputs");
585 return kFALSE;
586 }
587 }
588 }
589 }
590 // Process requested run numbers
a8739e8a 591 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 592 // Check validity of alien data directory
593 if (!fGridDataDir.Length()) {
594 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
595 return kFALSE;
596 }
923e2ca5 597 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 598 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
599 return kFALSE;
600 }
a2f5fc01 601 if (isCollection) {
c57f56b7 602 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
603 return kFALSE;
604 }
605
a2f5fc01 606 if (checked && !isXml) {
c57f56b7 607 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
608 return kFALSE;
609 }
610 // Check validity of run number(s)
611 TObjArray *arr;
612 TObjString *os;
74013535 613 TString format;
319593fb 614 Int_t nruns = 0;
904f9f5f 615 TString schunk, schunk2;
c57f56b7 616 TString path;
617 if (!checked) {
618 checked = kTRUE;
a2f5fc01 619 useTags = fDataPattern.Contains("tag");
620 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 621 }
a2f5fc01 622 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 623 Error("CheckInputData", "Cannot mix input files using/not using tags");
624 return kFALSE;
625 }
626 if (fRunNumbers.Length()) {
a8739e8a 627 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 628 arr = fRunNumbers.Tokenize(" ");
629 TIter next(arr);
630 while ((os=(TObjString*)next())) {
631 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 632 if (!DirectoryExists(path)) {
633 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 634 continue;
c57f56b7 635 }
636 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
637 TString msg = "\n##### file: ";
638 msg += path;
639 msg += " type: xml_collection;";
a2f5fc01 640 if (useTags) msg += " using_tags: Yes";
c57f56b7 641 else msg += " using_tags: No";
23329835 642 Info("CheckDataType", "%s", msg.Data());
319593fb 643 if (fNrunsPerMaster<2) {
d2a409b2 644 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 645 } else {
646 nruns++;
647 if (((nruns-1)%fNrunsPerMaster) == 0) {
648 schunk = os->GetString();
649 }
650 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
651 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 652 AddDataFile(schunk);
319593fb 653 }
c57f56b7 654 }
655 delete arr;
a8739e8a 656 } else {
657 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
658 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 659 format = Form("%%s/%s ", fRunPrefix.Data());
660 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 661 if (!DirectoryExists(path)) {
a8739e8a 662 continue;
663 }
74013535 664 format = Form("%%s/%s.xml", fRunPrefix.Data());
665 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 666 TString msg = "\n##### file: ";
667 msg += path;
668 msg += " type: xml_collection;";
a2f5fc01 669 if (useTags) msg += " using_tags: Yes";
a8739e8a 670 else msg += " using_tags: No";
23329835 671 Info("CheckDataType", "%s", msg.Data());
319593fb 672 if (fNrunsPerMaster<2) {
74013535 673 format = Form("%s.xml", fRunPrefix.Data());
674 AddDataFile(Form(format.Data(),irun));
319593fb 675 } else {
676 nruns++;
677 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 678 schunk = Form(fRunPrefix.Data(),irun);
319593fb 679 }
74013535 680 format = Form("_%s.xml", fRunPrefix.Data());
681 schunk2 = Form(format.Data(), irun);
319593fb 682 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 683 schunk += schunk2;
d2a409b2 684 AddDataFile(schunk);
319593fb 685 }
a8739e8a 686 }
904f9f5f 687 if (!fInputFiles) {
688 schunk += schunk2;
689 AddDataFile(schunk);
690 }
c57f56b7 691 }
692 return kTRUE;
693}
694
695//______________________________________________________________________________
696Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
697{
698// Create dataset for the grid data directory + run number.
d3b18c4c 699 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 700 if (!Connect()) {
701 Error("CreateDataset", "Cannot create dataset with no grid connection");
702 return kFALSE;
703 }
704
705 // Cd workspace
d3b18c4c 706 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 707 TString workdir = gGrid->GetHomeDirectory();
708 workdir += fGridWorkingDir;
709
710 // Compose the 'find' command arguments
74013535 711 TString format;
c57f56b7 712 TString command;
713 TString options = "-x collection ";
bb885a9e 714 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 715 TString conditions = "";
716
717 TString file;
718 TString path;
319593fb 719 Int_t nruns = 0;
904f9f5f 720 TString schunk, schunk2;
ab254fd1 721 TGridCollection *cbase=0, *cadd=0;
d2a409b2 722 if (!fRunNumbers.Length() && !fRunRange[0]) {
723 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
724 // Make a single data collection from data directory.
725 path = fGridDataDir;
923e2ca5 726 if (!DirectoryExists(path)) {
d2a409b2 727 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
728 return kFALSE;
729 }
923e2ca5 730// CdWork();
d2a409b2 731 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
732 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 733 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 734 command = "find ";
735 command += options;
736 command += path;
737 command += " ";
738 command += pattern;
739 command += conditions;
84fcd93f 740 printf("command: %s\n", command.Data());
d2a409b2 741 TGridResult *res = gGrid->Command(command);
742 if (res) delete res;
743 // Write standard output to file
744 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 745 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
746 Bool_t nullFile = kFALSE;
747 if (!hasGrep) {
d3339be3 748 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
749 } else {
a2f5fc01 750 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
751 if (nullFile) {
defd7a3a 752 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
753 return kFALSE;
754 }
d3339be3 755 }
0f389141 756 }
757 Bool_t fileExists = FileExists(file);
758 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 759 // Copy xml file to alien space
0f389141 760 if (fileExists) gGrid->Rm(file);
d2a409b2 761 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
762 if (!FileExists(file)) {
763 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
764 return kFALSE;
765 }
766 // Update list of files to be processed.
767 }
768 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
769 return kTRUE;
770 }
c57f56b7 771 // Several runs
a2f5fc01 772 Bool_t nullResult = kTRUE;
a8739e8a 773 if (fRunNumbers.Length()) {
774 TObjArray *arr = fRunNumbers.Tokenize(" ");
775 TObjString *os;
776 TIter next(arr);
777 while ((os=(TObjString*)next())) {
778 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 779 if (!DirectoryExists(path)) continue;
780// CdWork();
a8739e8a 781 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
782 else file = Form("%s.xml", os->GetString().Data());
319593fb 783 // If local collection file does not exist, create it via 'find' command.
d3339be3 784 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 785 command = "find ";
786 command += options;
787 command += path;
788 command += pattern;
789 command += conditions;
790 TGridResult *res = gGrid->Command(command);
791 if (res) delete res;
792 // Write standard output to file
793 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 794 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
795 Bool_t nullFile = kFALSE;
796 if (!hasGrep) {
d3339be3 797 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
798 } else {
a2f5fc01 799 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
800 if (nullFile) {
defd7a3a 801 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
802 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
803 continue;
804 }
d3339be3 805 }
a2f5fc01 806 nullResult = kFALSE;
d3339be3 807 }
a8739e8a 808 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 809 // Check if there is one run per master job.
810 if (fNrunsPerMaster<2) {
811 if (FileExists(file)) {
0f389141 812 if (fOverwriteMode) gGrid->Rm(file);
813 else {
814 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
815 continue;
816 }
319593fb 817 }
818 // Copy xml file to alien space
819 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
820 if (!FileExists(file)) {
821 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
822 delete arr;
823 return kFALSE;
824 }
825 } else {
826 nruns++;
827 if (((nruns-1)%fNrunsPerMaster) == 0) {
828 schunk = os->GetString();
829 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
830 } else {
831 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 832 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 833 cbase->Add(cadd);
834 delete cadd;
835 }
836 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
837 continue;
838 }
839 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 840 if (FileExists(schunk)) {
841 if (fOverwriteMode) gGrid->Rm(file);
842 else {
843 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
844 continue;
845 }
319593fb 846 }
84fcd93f 847 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 848 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 849 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 850 if (!FileExists(schunk)) {
851 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
852 delete arr;
853 return kFALSE;
854 }
d3339be3 855 }
a8739e8a 856 }
857 delete arr;
a2f5fc01 858 if (nullResult) {
d3339be3 859 Error("CreateDataset", "No valid dataset corresponding to the query!");
860 return kFALSE;
861 }
a8739e8a 862 } else {
863 // Process a full run range.
864 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 865 format = Form("%%s/%s ", fRunPrefix.Data());
866 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 867 if (!DirectoryExists(path)) continue;
868// CdWork();
74013535 869 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 870 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 871 else file = Form(format.Data(), irun);
0f389141 872 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
873 if (fOverwriteMode) gGrid->Rm(file);
874 else {
875 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
876 continue;
877 }
a8739e8a 878 }
319593fb 879 // If local collection file does not exist, create it via 'find' command.
d3339be3 880 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 881 command = "find ";
882 command += options;
883 command += path;
884 command += pattern;
885 command += conditions;
886 TGridResult *res = gGrid->Command(command);
887 if (res) delete res;
888 // Write standard output to file
889 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 890 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
891 Bool_t nullFile = kFALSE;
892 if (!hasGrep) {
d3339be3 893 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
894 } else {
a2f5fc01 895 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
896 if (nullFile) {
defd7a3a 897 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
898 continue;
899 }
d3339be3 900 }
a2f5fc01 901 nullResult = kFALSE;
319593fb 902 }
a8739e8a 903 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 904 // Check if there is one run per master job.
905 if (fNrunsPerMaster<2) {
906 if (FileExists(file)) {
0f389141 907 if (fOverwriteMode) gGrid->Rm(file);
908 else {
909 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
910 continue;
911 }
319593fb 912 }
913 // Copy xml file to alien space
914 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
915 if (!FileExists(file)) {
916 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
917 return kFALSE;
918 }
919 } else {
920 nruns++;
95e5b448 921 // Check if the collection for the chunk exist locally.
922 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 923 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
924 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
925 else continue;
926 }
84fcd93f 927 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 928 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 929 schunk = Form(fRunPrefix.Data(), irun);
319593fb 930 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 } else {
932 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
933 cbase->Add(cadd);
934 delete cadd;
935 }
74013535 936 format = Form("%%s_%s.xml", fRunPrefix.Data());
937 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 938 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 939 continue;
940 }
904f9f5f 941 schunk = schunk2;
319593fb 942 if (FileExists(schunk)) {
0f389141 943 if (fOverwriteMode) gGrid->Rm(schunk);
944 else {
945 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
946 continue;
947 }
319593fb 948 }
84fcd93f 949 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 950 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 951 if (FileExists(schunk)) {
0f389141 952 if (fOverwriteMode) gGrid->Rm(schunk);
953 else {
954 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
955 continue;
956 }
95e5b448 957 }
319593fb 958 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
959 if (!FileExists(schunk)) {
960 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
961 return kFALSE;
962 }
963 }
c57f56b7 964 }
a2f5fc01 965 if (nullResult) {
d3339be3 966 Error("CreateDataset", "No valid dataset corresponding to the query!");
967 return kFALSE;
968 }
a8739e8a 969 }
c57f56b7 970 return kTRUE;
971}
972
973//______________________________________________________________________________
974Bool_t AliAnalysisAlien::CreateJDL()
975{
976// Generate a JDL file according to current settings. The name of the file is
977// specified by fJDLName.
978 Bool_t error = kFALSE;
979 TObjArray *arr = 0;
980 Bool_t copy = kTRUE;
d3b18c4c 981 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 982 Bool_t generate = kTRUE;
983 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
984 if (!Connect()) {
985 Error("CreateJDL", "Alien connection required");
986 return kFALSE;
987 }
988 // Check validity of alien workspace
d3b18c4c 989 TString workdir;
990 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
991 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 992 workdir += fGridWorkingDir;
993 if (generate) {
994 TObjString *os;
995 if (!fInputFiles) {
996 Error("CreateJDL()", "Define some input files for your analysis.");
997 error = kTRUE;
998 }
999 // Compose list of input files
1000 // Check if output files were defined
1001 if (!fOutputFiles.Length()) {
1002 Error("CreateJDL", "You must define at least one output file");
1003 error = kTRUE;
1004 }
1005 // Check if an output directory was defined and valid
1006 if (!fGridOutputDir.Length()) {
1007 Error("CreateJDL", "You must define AliEn output directory");
1008 error = kTRUE;
1009 } else {
d3b18c4c 1010 if (!fProductionMode) {
1011 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1012 if (!DirectoryExists(fGridOutputDir)) {
1013 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1014 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1015 } else {
1016 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1017 // error = kTRUE;
1018 }
67a3b254 1019 } else {
1020 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1021 }
1022 gGrid->Cd(workdir);
1023 }
c57f56b7 1024 }
1025 // Exit if any error up to now
1026 if (error) return kFALSE;
1027 // Set JDL fields
0f389141 1028 if (!fUser.IsNull()) {
1029 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1030 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1031 }
1032 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1033 TString mergeExec = fExecutable;
1034 mergeExec.ReplaceAll(".sh", "_merge.sh");
1035 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1036 mergeExec.ReplaceAll(".sh", ".C");
1037 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1038 if (!fArguments.IsNull())
1039 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1040 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1041 else {
1042 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1043 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1044 }
b3e07543 1045
1f0d1ca2 1046 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1049 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1050
0f389141 1051 if (fMaxInitFailed > 0) {
c57f56b7 1052 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1053 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1054 }
1055 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1056 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1057 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1058 }
1059 if (!IsOneStageMerging()) {
f47d5cb4 1060 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1061 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1062 }
1063 if (fSplitMode.Length()) {
c57f56b7 1064 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1065 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1066 }
1067 fMergingJDL->SetValue("Split", "\"se\"");
1068 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1069 if (!fAliROOTVersion.IsNull()) {
1070 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1071 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1072 }
1073 if (!fROOTVersion.IsNull()) {
c57f56b7 1074 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1075 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1076 }
1077 if (!fAPIVersion.IsNull()) {
c57f56b7 1078 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1079 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1080 }
648174cf 1081 if (!fExternalPackages.IsNull()) {
1082 arr = fExternalPackages.Tokenize(" ");
1083 TIter next(arr);
1084 while ((os=(TObjString*)next())) {
1085 TString pkgname = os->GetString();
1086 Int_t index = pkgname.Index("::");
1087 TString pkgversion = pkgname(index+2, pkgname.Length());
1088 pkgname.Remove(index);
1089 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1090 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1091 }
1092 delete arr;
1093 }
0f389141 1094 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1096 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1097 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1098 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1099 TString analysisFile = fExecutable;
1100 analysisFile.ReplaceAll(".sh", ".root");
1101 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1102 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1103 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1104 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1105 if (fAdditionalLibs.Length()) {
1106 arr = fAdditionalLibs.Tokenize(" ");
1107 TIter next(arr);
1108 while ((os=(TObjString*)next())) {
1109 if (os->GetString().Contains(".so")) continue;
1110 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1111 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1112 }
1113 delete arr;
1114 }
4e5c5506 1115 if (fPackages) {
1116 TIter next(fPackages);
1117 TObject *obj;
0f389141 1118 while ((obj=next())) {
4e5c5506 1119 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1120 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1121 }
4e5c5506 1122 }
c57f56b7 1123 if (fOutputArchive.Length()) {
1124 arr = fOutputArchive.Tokenize(" ");
1125 TIter next(arr);
0f389141 1126 Bool_t first = kTRUE;
1127 const char *comment = "Files to be archived";
1128 const char *comment1 = comment;
1129 while ((os=(TObjString*)next())) {
1130 if (!first) comment = NULL;
1131 if (!os->GetString().Contains("@") && fCloseSE.Length())
1132 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1133 else
1134 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1135 first = kFALSE;
1136 }
c57f56b7 1137 delete arr;
f790bc1b 1138 // Output archive for the merging jdl
1139 TString outputArchive;
1140 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1141 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1142 // Add normal output files, extra files + terminate files
1143 TString files = GetListOfFiles("outextter");
1144 // Do not register merge excludes
1145 if (!fMergeExcludes.IsNull()) {
1146 arr = fMergeExcludes.Tokenize(" ");
1147 TIter next1(arr);
1148 while ((os=(TObjString*)next1())) {
1149 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1150 files.ReplaceAll(os->GetString(),"");
1151 }
66d71516 1152 delete arr;
0f389141 1153 }
f790bc1b 1154 files.ReplaceAll(".root", "*.root");
b83f84f0 1155 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1156 } else {
95f84779 1157 TString files = fOutputArchive;
1158 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1159 outputArchive = files;
f790bc1b 1160 }
0f389141 1161 arr = outputArchive.Tokenize(" ");
705adb3e 1162 TIter next2(arr);
0f389141 1163 comment = comment1;
1164 first = kTRUE;
705adb3e 1165 while ((os=(TObjString*)next2())) {
0f389141 1166 if (!first) comment = NULL;
0cdf65a8 1167 TString currentfile = os->GetString();
0cdf65a8 1168 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1169 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1170 else
0cdf65a8 1171 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1172 first = kFALSE;
1173 }
1174 delete arr;
c57f56b7 1175 }
149d288c 1176 arr = fOutputFiles.Tokenize(",");
c57f56b7 1177 TIter next(arr);
0f389141 1178 Bool_t first = kTRUE;
f790bc1b 1179 const char *comment = "Files to be saved";
43da816a 1180 while ((os=(TObjString*)next())) {
e1eaf596 1181 // Ignore ouputs in jdl that are also in outputarchive
1182 TString sout = os->GetString();
f790bc1b 1183 sout.ReplaceAll("*", "");
1184 sout.ReplaceAll(".root", "");
e1eaf596 1185 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1186 if (fOutputArchive.Contains(sout)) continue;
0f389141 1187 if (!first) comment = NULL;
43da816a 1188 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1189 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1190 else
0f389141 1191 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1192 first = kFALSE;
f790bc1b 1193 if (fMergeExcludes.Contains(sout)) continue;
1194 if (!os->GetString().Contains("@") && fCloseSE.Length())
1195 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1196 else
1197 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1198 }
c57f56b7 1199 delete arr;
0f389141 1200 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1201 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1202 TString validationScript = fValidationScript;
0f389141 1203 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1204 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1205 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1206 if (fMasterResubmitThreshold) {
1207 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1208 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1209 }
d2a409b2 1210 // Write a jdl with 2 input parameters: collection name and output dir name.
1211 WriteJDL(copy);
c57f56b7 1212 }
1213 // Copy jdl to grid workspace
a8739e8a 1214 if (copy) {
b5e4aaa7 1215 // Check if an output directory was defined and valid
1216 if (!fGridOutputDir.Length()) {
1217 Error("CreateJDL", "You must define AliEn output directory");
1218 return kFALSE;
1219 } else {
1220 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1221 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1222 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1223 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1224 } else {
1225 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1226 return kFALSE;
1227 }
1228 }
1229 gGrid->Cd(workdir);
1230 }
648174cf 1231 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1232 TString mergeJDLName = fExecutable;
1233 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1234 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1235 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1236 if (fProductionMode) {
648174cf 1237 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1238 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1239 }
648174cf 1240 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1241 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1242 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1243 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1244 if (fMergeViaJDL) {
1245 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1246 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1247 }
648174cf 1248 }
c57f56b7 1249 if (fAdditionalLibs.Length()) {
e7c71df0 1250 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1251 TObjString *os;
1252 TIter next(arr);
1253 while ((os=(TObjString*)next())) {
c57f56b7 1254 if (os->GetString().Contains(".so")) continue;
4e5c5506 1255 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1256 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1257 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1258 }
1259 delete arr;
1260 }
4e5c5506 1261 if (fPackages) {
1262 TIter next(fPackages);
1263 TObject *obj;
1264 while ((obj=next())) {
fdbbc7be 1265 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1266 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1267 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1268 }
1269 }
c57f56b7 1270 }
1271 return kTRUE;
1272}
1273
a8739e8a 1274//______________________________________________________________________________
d2a409b2 1275Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1276{
1277// Writes one or more JDL's corresponding to findex. If findex is negative,
1278// all run numbers are considered in one go (jdl). For non-negative indices
1279// they correspond to the indices in the array fInputFiles.
1280 if (!fInputFiles) return kFALSE;
d3b18c4c 1281 TObject *os;
1282 TString workdir;
1283 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1284 workdir += fGridWorkingDir;
f47d5cb4 1285 TString stageName = "$2";
1286 if (fProductionMode) stageName = "$4";
1287 if (!fMergeDirName.IsNull()) {
1288 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1289 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1290 } else {
1291 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1292 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1293 }
d3b18c4c 1294 if (fProductionMode) {
a8739e8a 1295 TIter next(fInputFiles);
b3e07543 1296 while ((os=next())) {
d3b18c4c 1297 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1298 }
d3b18c4c 1299 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1300 } else {
1301 if (!fRunNumbers.Length() && !fRunRange[0]) {
1302 // One jdl with no parameters in case input data is specified by name.
1303 TIter next(fInputFiles);
1304 while ((os=next()))
1305 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1306 if (!fOutputSingle.IsNull())
1307 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1308 else {
1309 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1310 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1311 }
1312 } else {
1313 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1314 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1315 if (!fOutputSingle.IsNull()) {
1316 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1317 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1318 } else {
1319 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1320 }
1321 }
a8739e8a 1322 }
1323
a8739e8a 1324 // Generate the JDL as a string
1325 TString sjdl = fGridJDL->Generate();
0f389141 1326 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1327 // Final merge jdl
1328 if (!fMergeDirName.IsNull()) {
1329 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1330 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1331 } else {
1332 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1333 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1334 }
b3e07543 1335 TString sjdl2 = fMergingJDL->Generate();
1336 Int_t index, index1;
a8739e8a 1337 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1338 sjdl.ReplaceAll("(member", "\n (member");
1339 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1340 sjdl.ReplaceAll("{", "{\n ");
1341 sjdl.ReplaceAll("};", "\n};");
1342 sjdl.ReplaceAll("{\n \n", "{\n");
1343 sjdl.ReplaceAll("\n\n", "\n");
1344 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1345 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1346 sjdl1.ReplaceAll("(member", "\n (member");
1347 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1348 sjdl1.ReplaceAll("{", "{\n ");
1349 sjdl1.ReplaceAll("};", "\n};");
1350 sjdl1.ReplaceAll("{\n \n", "{\n");
1351 sjdl1.ReplaceAll("\n\n", "\n");
1352 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1353 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1354 sjdl2.ReplaceAll("(member", "\n (member");
1355 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1356 sjdl2.ReplaceAll("{", "{\n ");
1357 sjdl2.ReplaceAll("};", "\n};");
1358 sjdl2.ReplaceAll("{\n \n", "{\n");
1359 sjdl2.ReplaceAll("\n\n", "\n");
1360 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1361 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1362 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1363 index = sjdl.Index("JDLVariables");
1364 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1365 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1366 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1367 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1368 index = fJobTag.Index(":");
1369 if (index < 0) index = fJobTag.Length();
1370 TString jobTag = fJobTag;
f47d5cb4 1371 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1372 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1373 if (fProductionMode) {
1374 sjdl1.Prepend("# Generated merging jdl (production mode) \
1375 \n# $1 = full alien path to output directory to be merged \
1376 \n# $2 = train number \
1377 \n# $3 = production (like LHC10b) \
1378 \n# $4 = merging stage \
1379 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1380 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1381 sjdl2.Prepend("# Generated merging jdl \
1382 \n# $1 = full alien path to output directory to be merged \
1383 \n# $2 = train number \
1384 \n# $3 = production (like LHC10b) \
1385 \n# $4 = merging stage \
1386 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1387 } else {
1388 sjdl1.Prepend("# Generated merging jdl \
1389 \n# $1 = full alien path to output directory to be merged \
1390 \n# $2 = merging stage \
03335324 1391 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1392 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1393 sjdl2.Prepend("# Generated merging jdl \
1394 \n# $1 = full alien path to output directory to be merged \
1395 \n# $2 = merging stage \
03335324 1396 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1397 }
0f389141 1398 index = sjdl1.Index("JDLVariables");
1399 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1400 index = sjdl2.Index("JDLVariables");
1401 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1402 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1403 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1404 index = sjdl2.Index("Split =");
1405 if (index>=0) {
1406 index1 = sjdl2.Index("\n", index);
1407 sjdl2.Remove(index, index1-index+1);
1408 }
1409 index = sjdl2.Index("SplitMaxInputFileNumber");
1410 if (index>=0) {
1411 index1 = sjdl2.Index("\n", index);
1412 sjdl2.Remove(index, index1-index+1);
1413 }
1414 index = sjdl2.Index("InputDataCollection");
1415 if (index>=0) {
1416 index1 = sjdl2.Index(";", index);
1417 sjdl2.Remove(index, index1-index+1);
1418 }
1419 index = sjdl2.Index("InputDataListFormat");
1420 if (index>=0) {
1421 index1 = sjdl2.Index("\n", index);
1422 sjdl2.Remove(index, index1-index+1);
1423 }
1424 index = sjdl2.Index("InputDataList");
1425 if (index>=0) {
1426 index1 = sjdl2.Index("\n", index);
1427 sjdl2.Remove(index, index1-index+1);
1428 }
f47d5cb4 1429 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1430 // Write jdl to file
a8739e8a 1431 ofstream out;
d2a409b2 1432 out.open(fJDLName.Data(), ios::out);
a8739e8a 1433 if (out.bad()) {
d3b18c4c 1434 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1435 return kFALSE;
1436 }
1437 out << sjdl << endl;
b3e07543 1438 out.close();
0f389141 1439 TString mergeJDLName = fExecutable;
1440 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1441 if (fMergeViaJDL) {
1442 ofstream out1;
1443 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1444 if (out1.bad()) {
d3b18c4c 1445 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1446 return kFALSE;
1447 }
1448 out1 << sjdl1 << endl;
b3e07543 1449 out1.close();
1450 ofstream out2;
1451 TString finalJDL = mergeJDLName;
1452 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1453 out2.open(finalJDL.Data(), ios::out);
1454 if (out2.bad()) {
1455 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1456 return kFALSE;
1457 }
1458 out2 << sjdl2 << endl;
1459 out2.close();
0f389141 1460 }
a8739e8a 1461
1462 // Copy jdl to grid workspace
1463 if (!copy) {
d3b18c4c 1464 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1465 } else {
d2a409b2 1466 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1467 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1468 TString finalJDL = mergeJDLName;
1469 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1470 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1471 if (fProductionMode) {
b5fe9cba 1472 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1473 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1474 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1475 }
d2a409b2 1476 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1477 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1478 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1479 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1480 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1481 if (fMergeViaJDL) {
b3e07543 1482 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1483 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1484 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1485 }
a8739e8a 1486 }
1487 return kTRUE;
1488}
1489
c57f56b7 1490//______________________________________________________________________________
5513444a 1491Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1492{
1493// Returns true if file exists.
5513444a 1494 if (!gGrid) return kFALSE;
2ea55496 1495 TString slfn = lfn;
1496 slfn.ReplaceAll("alien://","");
1497 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1498 if (!res) return kFALSE;
1499 TMap *map = dynamic_cast<TMap*>(res->At(0));
1500 if (!map) {
1501 delete res;
1502 return kFALSE;
1503 }
1504 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1505 if (!objs || !objs->GetString().Length()) {
1506 delete res;
1507 return kFALSE;
1508 }
1509 delete res;
1510 return kTRUE;
1511}
1512
923e2ca5 1513//______________________________________________________________________________
1514Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1515{
1516// Returns true if directory exists. Can be also a path.
1517 if (!gGrid) return kFALSE;
1518 // Check if dirname is a path
1519 TString dirstripped = dirname;
1520 dirstripped = dirstripped.Strip();
1521 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1522 TString dir = gSystem->BaseName(dirstripped);
1523 dir += "/";
1524 TString path = gSystem->DirName(dirstripped);
1525 TGridResult *res = gGrid->Ls(path, "-F");
1526 if (!res) return kFALSE;
1527 TIter next(res);
1528 TMap *map;
1529 TObject *obj;
1530 while ((map=dynamic_cast<TMap*>(next()))) {
1531 obj = map->GetValue("name");
1532 if (!obj) break;
1533 if (dir == obj->GetName()) {
1534 delete res;
1535 return kTRUE;
1536 }
1537 }
1538 delete res;
1539 return kFALSE;
1540}
1541
c57f56b7 1542//______________________________________________________________________________
a2f5fc01 1543void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1544{
1545// Check input data type.
a2f5fc01 1546 isCollection = kFALSE;
1547 isXml = kFALSE;
1548 useTags = kFALSE;
c57f56b7 1549 if (!gGrid) {
1550 Error("CheckDataType", "No connection to grid");
1551 return;
1552 }
a2f5fc01 1553 isCollection = IsCollection(lfn);
c57f56b7 1554 TString msg = "\n##### file: ";
1555 msg += lfn;
a2f5fc01 1556 if (isCollection) {
c57f56b7 1557 msg += " type: raw_collection;";
1558 // special treatment for collections
a2f5fc01 1559 isXml = kFALSE;
c57f56b7 1560 // check for tag files in the collection
1561 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1562 if (!res) {
1563 msg += " using_tags: No (unknown)";
23329835 1564 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1565 return;
1566 }
1567 const char* typeStr = res->GetKey(0, "origLFN");
1568 if (!typeStr || !strlen(typeStr)) {
1569 msg += " using_tags: No (unknown)";
23329835 1570 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1571 return;
1572 }
1573 TString file = typeStr;
a2f5fc01 1574 useTags = file.Contains(".tag");
1575 if (useTags) msg += " using_tags: Yes";
c57f56b7 1576 else msg += " using_tags: No";
23329835 1577 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1578 return;
1579 }
1580 TString slfn(lfn);
1581 slfn.ToLower();
a2f5fc01 1582 isXml = slfn.Contains(".xml");
1583 if (isXml) {
c57f56b7 1584 // Open xml collection and check if there are tag files inside
1585 msg += " type: xml_collection;";
1586 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1587 if (!coll) {
1588 msg += " using_tags: No (unknown)";
23329835 1589 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1590 return;
1591 }
1592 TMap *map = coll->Next();
1593 if (!map) {
1594 msg += " using_tags: No (unknown)";
23329835 1595 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1596 return;
1597 }
1598 map = (TMap*)map->GetValue("");
1599 TString file;
1600 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1601 useTags = file.Contains(".tag");
c57f56b7 1602 delete coll;
a2f5fc01 1603 if (useTags) msg += " using_tags: Yes";
c57f56b7 1604 else msg += " using_tags: No";
23329835 1605 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1606 return;
1607 }
a2f5fc01 1608 useTags = slfn.Contains(".tag");
c57f56b7 1609 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1610 else msg += " type: unknown file;";
a2f5fc01 1611 if (useTags) msg += " using_tags: Yes";
c57f56b7 1612 else msg += " using_tags: No";
23329835 1613 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1614}
1615
4e5c5506 1616//______________________________________________________________________________
1617void AliAnalysisAlien::EnablePackage(const char *package)
1618{
1619// Enables a par file supposed to exist in the current directory.
1620 TString pkg(package);
1621 pkg.ReplaceAll(".par", "");
1622 pkg += ".par";
1623 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1624 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1625 return;
1626 }
1627 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1628 Info("EnablePackage", "AliEn plugin will use .par packages");
1629 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1630 if (!fPackages) {
1631 fPackages = new TObjArray();
1632 fPackages->SetOwner();
1633 }
1634 fPackages->Add(new TObjString(pkg));
1635}
1636
3bdcb562 1637//______________________________________________________________________________
1638TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1639{
1640// Make a tree from files having the location specified in fFileForTestMode.
1641// Inspired from JF's CreateESDChain.
1642 if (fFileForTestMode.IsNull()) {
1643 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1644 return NULL;
1645 }
1646 if (gSystem->AccessPathName(fFileForTestMode)) {
1647 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1648 return NULL;
1649 }
1650 // Open the file
1651 ifstream in;
1652 in.open(fFileForTestMode);
1653 Int_t count = 0;
1654 // Read the input list of files and add them to the chain
1655 TString line;
1656 TChain *chain = new TChain(treeName);
1657 while (in.good())
1658 {
1659 in >> line;
1660 if (line.IsNull()) continue;
1661 if (count++ == fNtestFiles) break;
1662 TString esdFile(line);
1663 TFile *file = TFile::Open(esdFile);
1664 if (file) {
1665 if (!file->IsZombie()) chain->Add(esdFile);
1666 file->Close();
1667 } else {
1668 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1669 }
1670 }
1671 in.close();
1672 if (!chain->GetListOfFiles()->GetEntries()) {
1673 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1674 delete chain;
1675 return NULL;
1676 }
1677// chain->ls();
1678 return chain;
1679}
1680
d2a409b2 1681//______________________________________________________________________________
1682const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1683{
1684// Get job status for all jobs with jobid>jobidstart.
1685 static char mstatus[20];
1686 mstatus[0] = '\0';
1687 nrunning = 0;
1688 nwaiting = 0;
1689 nerror = 0;
1690 ndone = 0;
1691 TGridJobStatusList *list = gGrid->Ps("");
1692 if (!list) return mstatus;
1693 Int_t nentries = list->GetSize();
1694 TGridJobStatus *status;
1695 Int_t pid;
1696 for (Int_t ijob=0; ijob<nentries; ijob++) {
1697 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1698 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1699 if (pid<jobidstart) continue;
1700 if (pid == lastid) {
8ddad121 1701 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1702 }
1703 switch (status->GetStatus()) {
1704 case TGridJobStatus::kWAITING:
1705 nwaiting++; break;
1706 case TGridJobStatus::kRUNNING:
1707 nrunning++; break;
1708 case TGridJobStatus::kABORTED:
1709 case TGridJobStatus::kFAIL:
1710 case TGridJobStatus::kUNKNOWN:
1711 nerror++; break;
1712 case TGridJobStatus::kDONE:
1713 ndone++;
1714 }
1715 }
1716 list->Delete();
1717 delete list;
1718 return mstatus;
1719}
1720
c57f56b7 1721//______________________________________________________________________________
1722Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1723{
1724// Returns true if file is a collection. Functionality duplicated from
1725// TAlien::Type() because we don't want to directly depend on TAlien.
1726 if (!gGrid) {
1727 Error("IsCollection", "No connection to grid");
1728 return kFALSE;
1729 }
1730 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1731 if (!res) return kFALSE;
1732 const char* typeStr = res->GetKey(0, "type");
1733 if (!typeStr || !strlen(typeStr)) return kFALSE;
1734 if (!strcmp(typeStr, "collection")) return kTRUE;
1735 delete res;
1736 return kFALSE;
1737}
1738
fe2d7fc2 1739//______________________________________________________________________________
1740Bool_t AliAnalysisAlien::IsSingleOutput() const
1741{
1742// Check if single-ouput option is on.
1743 return (!fOutputSingle.IsNull());
1744}
1745
16a4353c 1746//______________________________________________________________________________
1747void AliAnalysisAlien::Print(Option_t *) const
1748{
1749// Print current plugin settings.
84fcd93f 1750 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1751 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1752 if (mgr && mgr->IsProofMode()) {
1753 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1754 if (TestBit(AliAnalysisGrid::kTest))
1755 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1756 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1757 if (!fProofDataSet.IsNull())
1758 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1759 if (fProofReset==1)
1760 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (fProofReset>1)
1762 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1763 if (!fRootVersionForProof.IsNull())
1764 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1765 else
1766 printf("= ROOT version requested________________________ default\n");
1767 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1768 if (!fAliRootMode.IsNull())
1769 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1770 if (fNproofWorkers)
1771 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1772 if (fNproofWorkersPerSlave)
1773 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1774 if (TestSpecialBit(kClearPackages))
1775 printf("= ClearPackages requested...\n");
1776 if (fIncludePath.Data())
1777 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1778 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1779 if (fPackages && fPackages->GetEntries()) {
1780 TIter next(fPackages);
1781 TObject *obj;
1782 TString list;
1783 while ((obj=next())) list += obj->GetName();
1784 printf("= Par files to be used: ________________________ %s\n", list.Data());
1785 }
1786 if (TestSpecialBit(kProofConnectGrid))
1787 printf("= Requested PROOF connection to grid\n");
1788 return;
1789 }
e1c22e21 1790 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1791 if (fOverwriteMode) {
1792 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1793 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1794 }
348be253 1795 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1796 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1797 printf("= Production mode:______________________________ %d\n", fProductionMode);
1798 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1799 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1800 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1801 if (fUser.Length())
84fcd93f 1802 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1803 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1804 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1805 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1806 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1807 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1808 if (fRunNumbers.Length())
84fcd93f 1809 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1810 if (fRunRange[0])
74013535 1811 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1812 if (!fRunRange[0] && !fRunNumbers.Length()) {
1813 TIter next(fInputFiles);
1814 TObject *obj;
1815 TString list;
1816 while ((obj=next())) list += obj->GetName();
84fcd93f 1817 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1818 }
1819 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1820 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1821 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1822 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1823 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1824 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1825 printf("=====================================================================\n");
1826 printf("= Job price: ___________________________________ %d\n", fPrice);
1827 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1828 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1829 if (fMaxInitFailed>0)
84fcd93f 1830 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1831 if (fMasterResubmitThreshold>0)
84fcd93f 1832 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1833 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1834 if (fNrunsPerMaster>0)
84fcd93f 1835 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1836 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1837 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1838 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1839 if (fArguments.Length())
84fcd93f 1840 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1841 if (fExecutableArgs.Length())
1842 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1843 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1844 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1845 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1846 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1847 if (fDatasetName)
84fcd93f 1848 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1849 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1850 if (fIncludePath.Data())
84fcd93f 1851 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1852 if (fCloseSE.Length())
84fcd93f 1853 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1854 if (fFriendChainName.Length())
84fcd93f 1855 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1856 if (fPackages && fPackages->GetEntries()) {
16a4353c 1857 TIter next(fPackages);
1858 TObject *obj;
1859 TString list;
1860 while ((obj=next())) list += obj->GetName();
84fcd93f 1861 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1862 }
1863}
1864
c57f56b7 1865//______________________________________________________________________________
1866void AliAnalysisAlien::SetDefaults()
1867{
1868// Set default values for everything. What cannot be filled will be left empty.
1869 if (fGridJDL) delete fGridJDL;
1870 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1871 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1872 fPrice = 1;
1873 fTTL = 30000;
1874 fSplitMaxInputFileNumber = 100;
1875 fMaxInitFailed = 0;
1876 fMasterResubmitThreshold = 0;
bb885a9e 1877 fNtestFiles = 10;
149d288c 1878 fNreplicas = 2;
a8739e8a 1879 fRunRange[0] = 0;
1880 fRunRange[1] = 0;
8e3bd06a 1881 fRunPrefix = "%d";
319593fb 1882 fNrunsPerMaster = 1;
16a4353c 1883 fMaxMergeFiles = 100;
c57f56b7 1884 fRunNumbers = "";
1885 fExecutable = "analysis.sh";
0a1c1f7f 1886 fExecutableCommand = "root -b -q";
c57f56b7 1887 fArguments = "";
631c0b05 1888 fExecutableArgs = "";
c57f56b7 1889 fAnalysisMacro = "myAnalysis.C";
1890 fAnalysisSource = "";
1891 fAdditionalLibs = "";
1892 fSplitMode = "se";
1893 fAPIVersion = "";
1894 fROOTVersion = "";
1895 fAliROOTVersion = "";
1896 fUser = ""; // Your alien user name
1897 fGridWorkingDir = "";
1898 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1899 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1900 fFriendChainName = "";
c57f56b7 1901 fGridOutputDir = "output";
b83f84f0 1902 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1903 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1904 fInputFormat = "xml-single";
1905 fJDLName = "analysis.jdl";
c6cb3634 1906 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1907 fMergeExcludes = "";
0f389141 1908 fMergeViaJDL = 0;
348be253 1909 SetUseCopy(kTRUE);
1910 SetCheckCopy(kTRUE);
149d288c 1911 SetDefaultOutputs(kTRUE);
e1c22e21 1912 fOverwriteMode = 1;
c57f56b7 1913}
1914
0f389141 1915//______________________________________________________________________________
b3e07543 1916Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1917{
b3e07543 1918// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1919 // First check if the result is already in the output directory.
1920 if (FileExists(Form("%s/%s",aliendir,filename))) {
1921 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1922 return kFALSE;
1923 }
b3e07543 1924 // Now check the last stage done.
1925 Int_t stage = 0;
1926 while (1) {
1927 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1928 stage++;
1929 }
1930 // Next stage of merging
1931 stage++;
1932 TString pattern = "*root_archive.zip";
1933 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1934 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1935 if (res) delete res;
1936 // Write standard output to file
1937 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1938 // Count the number of files inside
1939 ifstream ifile;
1940 ifile.open(Form("Stage_%d.xml",stage));
1941 if (!ifile.good()) {
1942 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1943 return kFALSE;
b3e07543 1944 }
1945 TString line;
1946 Int_t nfiles = 0;
1947 while (!ifile.eof()) {
1948 ifile >> line;
1949 if (line.Contains("/event")) nfiles++;
7c2cd90a 1950 }
b3e07543 1951 ifile.close();
1952 if (!nfiles) {
1953 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1954 return kFALSE;
1955 } else {
1956 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1957 }
b3e07543 1958 // Copy the file in the output directory
1959 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1960 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1961 // Check if this is the last stage to be done.
1962 Bool_t laststage = (nfiles<nperchunk);
1963 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1964 if (laststage) {
1965 printf("### Submiting final merging stage %d\n", stage);
1966 TString finalJDL = jdl;
1967 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1968 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1969 Int_t jobId = SubmitSingleJob(query);
1970 if (!jobId) return kFALSE;
1971 } else {
1972 printf("### Submiting merging stage %d\n", stage);
03335324 1973 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1974 Int_t jobId = SubmitSingleJob(query);
1975 if (!jobId) return kFALSE;
7c2cd90a 1976 }
b3e07543 1977 return kTRUE;
1978}
7c2cd90a 1979
1980//______________________________________________________________________________
1981Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1982{
1983// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1984 if (!gGrid) return 0;
1985 printf("=> %s ------> ",query);
1986 TGridResult *res = gGrid->Command(query);
1987 if (!res) return 0;
1988 TString jobId = res->GetKey(0,"jobId");
1989 delete res;
1990 if (jobId.IsNull()) {
1991 printf("submission failed. Reason:\n");
1992 gGrid->Stdout();
1993 gGrid->Stderr();
1994 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1995 return 0;
1996 }
1997 printf(" Job id: %s\n", jobId.Data());
1998 return atoi(jobId);
1999}
2000
2001//______________________________________________________________________________
b3e07543 2002Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2003{
b3e07543 2004// Merge given output files from basedir. Basedir can be an alien output directory
2005// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2006// files in a group (ignored for xml input). Merging can be done in stages:
2007// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2008// stage=1 : works with an xml of all root_archive.zip in the output directory
2009// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2010 TString outputFile = output;
0f389141 2011 TString command;
a2f5fc01 2012 TString outputChunk;
2013 TString previousChunk = "";
b3e07543 2014 TObjArray *listoffiles = new TObjArray();
2015// listoffiles->SetOwner();
a2f5fc01 2016 Int_t countChunk = 0;
2017 Int_t countZero = nmaxmerge;
0f389141 2018 Bool_t merged = kTRUE;
a2f5fc01 2019 Int_t index = outputFile.Index("@");
2020 if (index > 0) outputFile.Remove(index);
7c2cd90a 2021 TString inputFile = outputFile;
b3e07543 2022 TString sbasedir = basedir;
2023 if (sbasedir.Contains(".xml")) {
2024 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2025 nmaxmerge = 9999999;
2026 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2027 if (!coll) {
2028 ::Error("MergeOutput", "Input XML collection empty.");
2029 return kFALSE;
2030 }
2031 // Iterate grid collection
2032 while (coll->Next()) {
2033 TString fname = gSystem->DirName(coll->GetTURL());
2034 fname += "/";
2035 fname += inputFile;
2036 listoffiles->Add(new TNamed(fname.Data(),""));
2037 }
2038 } else {
2039 command = Form("find %s/ *%s", basedir, inputFile.Data());
2040 printf("command: %s\n", command.Data());
2041 TGridResult *res = gGrid->Command(command);
2042 if (!res) {
2043 ::Error("MergeOutput","No result for the find command\n");
2044 delete listoffiles;
2045 return kFALSE;
2046 }
2047 TIter nextmap(res);
2048 TMap *map = 0;
2049 while ((map=(TMap*)nextmap())) {
2050 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2051 if (!objs || !objs->GetString().Length()) {
2052 // Nothing found - skip this output
2053 delete res;
2054 delete listoffiles;
2055 return kFALSE;
2056 }
2057 listoffiles->Add(new TNamed(objs->GetName(),""));
2058 }
2059 delete res;
2060 }
2061 if (!listoffiles->GetEntries()) {
7c2cd90a 2062 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2063 delete listoffiles;
0f389141 2064 return kFALSE;
2065 }
2066
2067 TFileMerger *fm = 0;
b3e07543 2068 TIter next0(listoffiles);
2069 TObjArray *listoffilestmp = new TObjArray();
2070 listoffilestmp->SetOwner();
2071 TObject *nextfile;
2072 TString snextfile;
2073 // Keep only the files at upper level
2074 Int_t countChar = 0;
2075 while ((nextfile=next0())) {
2076 snextfile = nextfile->GetName();
2077 Int_t crtCount = snextfile.CountChar('/');
2078 if (nextfile == listoffiles->First()) countChar = crtCount;
2079 if (crtCount < countChar) countChar = crtCount;
2080 }
2081 next0.Reset();
2082 while ((nextfile=next0())) {
2083 snextfile = nextfile->GetName();
2084 Int_t crtCount = snextfile.CountChar('/');
2085 if (crtCount > countChar) {
2086 delete nextfile;
2087 continue;
2088 }
2089 listoffilestmp->Add(nextfile);
2090 }
2091 delete listoffiles;
2092 listoffiles = listoffilestmp; // Now contains 'good' files
2093 listoffiles->Print();
2094 TIter next(listoffiles);
7c2cd90a 2095 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2096 outputChunk = outputFile;
2097 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2098 // Check for existent temporary merge files
e1c22e21 2099 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2100 // Preserve old merging functionality for stage 0.
2101 if (stage==0) {
2102 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2103 while (1) {
2104 // Skip as many input files as in a chunk
2ea55496 2105 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2106 nextfile = next();
2107 if (!nextfile) {
2ea55496 2108 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2109 delete listoffiles;
2ea55496 2110 return kFALSE;
2111 }
b3e07543 2112 snextfile = nextfile->GetName();
7c2cd90a 2113 }
2114 outputChunk = outputFile;
2115 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2116 countChunk++;
2117 if (gSystem->AccessPathName(outputChunk)) continue;
2118 // Merged file with chunks up to <countChunk> found
2119 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2120 previousChunk = outputChunk;
2121 break;
2122 }
2123 }
2124 countZero = nmaxmerge;
2125
b3e07543 2126 while ((nextfile=next())) {
2127 snextfile = nextfile->GetName();
2ea55496 2128 // Loop 'find' results and get next LFN
7c2cd90a 2129 if (countZero == nmaxmerge) {
2130 // First file in chunk - create file merger and add previous chunk if any.
2131 fm = new TFileMerger(kFALSE);
2132 fm->SetFastMethod(kTRUE);
2133 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2134 outputChunk = outputFile;
2135 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2136 }
2137 // If last file found, put merged results in the output file
b3e07543 2138 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2139 // Add file to be merged and decrement chunk counter.
b3e07543 2140 fm->AddFile(snextfile);
7c2cd90a 2141 countZero--;
b3e07543 2142 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2143 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2144 // Nothing found - skip this output
2145 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2146 merged = kFALSE;
2147 break;
7c2cd90a 2148 }
2149 fm->OutputFile(outputChunk);
2150 // Merge the outputs, then go to next chunk
2151 if (!fm->Merge()) {
2152 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2153 merged = kFALSE;
2154 break;
7c2cd90a 2155 } else {
2156 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2157 gSystem->Unlink(previousChunk);
2158 }
b3e07543 2159 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2160 countChunk++;
2161 countZero = nmaxmerge;
2162 previousChunk = outputChunk;
0f389141 2163 }
0f389141 2164 }
b3e07543 2165 delete listoffiles;
2166 delete fm;
7c2cd90a 2167 return merged;
2168 }
2169 // Merging stage different than 0.
2170 // Move to the begining of the requested chunk.
7c2cd90a 2171 fm = new TFileMerger(kFALSE);
2172 fm->SetFastMethod(kTRUE);
b3e07543 2173 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2174 delete listoffiles;
7c2cd90a 2175 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2176 // Nothing found - skip this output
2177 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2178 delete fm;
2179 return kFALSE;
2180 }
b3e07543 2181 fm->OutputFile(outputFile);
7c2cd90a 2182 // Merge the outputs
2183 if (!fm->Merge()) {
2184 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2185 delete fm;
2186 return kFALSE;
2187 } else {
b3e07543 2188 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2189 }
7c2cd90a 2190 delete fm;
2191 return kTRUE;
0f389141 2192}
2193
c57f56b7 2194//______________________________________________________________________________
2195Bool_t AliAnalysisAlien::MergeOutputs()
2196{
2197// Merge analysis outputs existing in the AliEn space.
2198 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2199 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2200 if (!Connect()) {
2201 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2202 return kFALSE;
0f389141 2203 }
c9e8f7fd 2204 if (fMergeViaJDL) {
2205 if (!TestBit(AliAnalysisGrid::kMerge)) {
2206 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2207 return kFALSE;
2208 }
2209 if (fProductionMode) {
2210 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2211 return kFALSE;
2212 }
0f389141 2213 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2214 if (!SubmitMerging()) return kFALSE;
0f389141 2215 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2216 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2217 return kFALSE;
c57f56b7 2218 }
2219 // Get the output path
d2a409b2 2220 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2221 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2222 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2223 return kFALSE;
2224 }
2225 if (!fOutputFiles.Length()) {
2226 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2227 return kFALSE;
0f389141 2228 }
2229 // Check if fast read option was requested
e1c22e21 2230 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2231 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2232 if (fFastReadOption) {
d3339be3 2233 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2234 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2235 gEnv->SetValue("XNet.ConnectTimeout",50);
2236 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2237 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2238 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2239 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2240 }
e8b839ab 2241 // Make sure we change the temporary directory
2242 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2243 // Set temporary compilation directory to current one
2244 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2245 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2246 TIter next(list);
2247 TObjString *str;
a2f5fc01 2248 TString outputFile;
c57f56b7 2249 Bool_t merged = kTRUE;
2250 while((str=(TObjString*)next())) {
a2f5fc01 2251 outputFile = str->GetString();
2252 Int_t index = outputFile.Index("@");
2253 if (index > 0) outputFile.Remove(index);
2254 TString outputChunk = outputFile;
2255 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2256 // Skip already merged outputs
a2f5fc01 2257 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2258 if (fOverwriteMode) {
a2f5fc01 2259 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2260 gSystem->Unlink(outputFile);
2261 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2262 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2263 outputChunk.Data());
2264 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2265 }
2266 } else {
a2f5fc01 2267 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2268 continue;
2269 }
2270 } else {
a2f5fc01 2271 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2272 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2273 outputChunk.Data());
2274 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2275 }
2276 }
bb885a9e 2277 if (fMergeExcludes.Length() &&
a2f5fc01 2278 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2279 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2280 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2281 if (!merged) {
2282 Error("MergeOutputs", "Terminate() will NOT be executed");
2283 return kFALSE;
ff07ec61 2284 }
2285 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2286 if (fileOpened) fileOpened->Close();
c57f56b7 2287 }
0f389141 2288 return kTRUE;
c57f56b7 2289}
2290
bb885a9e 2291//______________________________________________________________________________
2292void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2293{
2294// Use the output files connected to output containers from the analysis manager
2295// rather than the files defined by SetOutputFiles
2296 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2297 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2298 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2299}
2300
149d288c 2301//______________________________________________________________________________
2302void AliAnalysisAlien::SetOutputFiles(const char *list)
2303{
2304// Manually set the output files list.
2305// Removes duplicates. Not allowed if default outputs are not disabled.
2306 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2307 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2308 return;
2309 }
2310 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2311 fOutputFiles = "";
2312 TString slist = list;
2313 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2314 TObjArray *arr = slist.Tokenize(" ");
2315 TObjString *os;
2316 TIter next(arr);
2317 TString sout;
2318 while ((os=(TObjString*)next())) {
2319 sout = os->GetString();
2320 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2321 if (fOutputFiles.Contains(sout)) continue;
2322 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2323 fOutputFiles += sout;
2324 }
2325 delete arr;
f790bc1b 2326}
149d288c 2327
2328//______________________________________________________________________________
2329void AliAnalysisAlien::SetOutputArchive(const char *list)
2330{
2331// Manually set the output archive list. Free text - you are on your own...
2332// Not allowed if default outputs are not disabled.
2333 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2334 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2335 return;
2336 }
2337 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2338 fOutputArchive = list;
2339}
2340
2341//______________________________________________________________________________
2342void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2343{
2344// Setting a prefered output SE is not allowed anymore.
2345 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2346}
2347
5513444a 2348//______________________________________________________________________________
2349Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2350{
2351// Start remote grid analysis.
3bdcb562 2352 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2353 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2354 if (!mgr || !mgr->IsInitialized()) {
2355 Error("StartAnalysis", "You need an initialized analysis manager for this");
2356 return kFALSE;
2357 }
2358 // Are we in PROOF mode ?
2359 if (mgr->IsProofMode()) {
90d50a8c 2360 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2361 if (fProofCluster.IsNull()) {
2362 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2363 return kFALSE;
2364 }
2365 if (fProofDataSet.IsNull() && !testMode) {
2366 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2367 return kFALSE;
2368 }
2369 // Set the needed environment
2370 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2371 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2372 if (fProofReset && !testMode) {
2373 if (fProofReset==1) {
2374 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2375 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2376 } else {
2377 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2378 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2379 }
2380 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2381 return kFALSE;
2382 }
2383 // Do we need to change the ROOT version ? The success of this cannot be checked.
2384 if (!fRootVersionForProof.IsNull() && !testMode) {
2385 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2386 fProofCluster.Data(), fRootVersionForProof.Data()));
2387 }
2388 // Connect to PROOF and check the status
2389 Long_t proof = 0;
27734f0e 2390 TString sworkers;
2391 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2392 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2393 if (!testMode) {
27734f0e 2394 if (!sworkers.IsNull())
2395 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2396 else
2397 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2398 } else {
2399 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2400 if (!proof) {
2401 Error("StartAnalysis", "Could not start PROOF in test mode");
2402 return kFALSE;
2403 }
2404 }
2405 if (!proof) {
2406 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2407 return kFALSE;
2408 }
27734f0e 2409 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2410 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2411 // Is dataset existing ?
2412 if (!testMode) {
2413 TString dataset = fProofDataSet;
2414 Int_t index = dataset.Index("#");
2415 if (index>=0) dataset.Remove(index);
830acc4c 2416// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2417// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2418// return kFALSE;
2419// }
2420// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2421 }
2422 // Is ClearPackages() needed ?
2423 if (TestSpecialBit(kClearPackages)) {
2424 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2425 gROOT->ProcessLine("gProof->ClearPackages();");
2426 }
2427 // Is a given aliroot mode requested ?
2428 TList optionsList;
1ac846e9 2429 TString parLibs;
830acc4c 2430 if (!fAliRootMode.IsNull()) {
3bdcb562 2431 TString alirootMode = fAliRootMode;
2432 if (alirootMode == "default") alirootMode = "";
2433 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2434 optionsList.SetOwner();
2435 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2436 // Check the additional libs to be loaded
2437 TString extraLibs;
1ac846e9 2438 Bool_t parMode = kFALSE;
3bdcb562 2439 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2440 // Parse the extra libs for .so
2441 if (fAdditionalLibs.Length()) {
2442 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2443 TIter next(list);
2444 TObjString *str;
1ac846e9 2445 while((str=(TObjString*)next())) {
2446 if (str->GetString().Contains(".so")) {
2447 if (parMode) {
2448 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2449 break;
2450 }
2451 TString stmp = str->GetName();
2452 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2453 stmp.ReplaceAll(".so","");
2454 if (!extraLibs.IsNull()) extraLibs += ":";
2455 extraLibs += stmp;
2456 continue;
2457 }
2458 if (str->GetString().Contains(".par")) {
2459 // The first par file found in the list will not allow any further .so
2460 parMode = kTRUE;
2461 if (!parLibs.IsNull()) parLibs += ":";
2462 parLibs += str->GetName();
2463 continue;
2464 }
3bdcb562 2465 }
2466 if (list) delete list;
2467 }
2468 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2469 // Check extra includes
2470 if (!fIncludePath.IsNull()) {
2471 TString includePath = fIncludePath;
2472 includePath.ReplaceAll(" ",":");
8ddad121 2473 includePath.ReplaceAll("$ALICE_ROOT","");
2474 includePath.ReplaceAll("${ALICE_ROOT}","");
2475 includePath.ReplaceAll("-I","");
830acc4c 2476 includePath.Strip(TString::kTrailing, ':');
2477 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2478 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2479 }
27734f0e 2480 // Check if connection to grid is requested
2481 if (TestSpecialBit(kProofConnectGrid))
2482 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2483 // Enable AliRoot par
2484 if (testMode) {
2485 // Enable proof lite package
2486 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2487 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2488 TNamed *obj = (TNamed*)optionsList.At(i);
2489 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2490 }
2491 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2492 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2493 Info("StartAnalysis", "AliRootProofLite enabled");
2494 } else {
2495 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2496 return kFALSE;
2497 }
2498 } else {
8ddad121 2499 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2500 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2501 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2502 return kFALSE;
2503 }
27734f0e 2504 }
1ac846e9 2505 // Enable first par files from fAdditionalLibs
2506 if (!parLibs.IsNull()) {
2507 TObjArray *list = parLibs.Tokenize(":");
2508 TIter next(list);
2509 TObjString *package;
2510 while((package=(TObjString*)next())) {
40501bb7 2511 TString spkg = package->GetName();
2512 spkg.ReplaceAll(".par", "");
2513 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2514 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2515 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2516 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2517 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2518 return kFALSE;
2519 }
2520 } else {
2521 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2522 return kFALSE;
2523 }
2524 }
2525 if (list) delete list;
2526 }
3bdcb562 2527 } else {
2528 if (fAdditionalLibs.Contains(".so") && !testMode) {
2529 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2530 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2531 return kFALSE;
2532 }
2533 }
2534 // Enable par files if requested
2535 if (fPackages && fPackages->GetEntries()) {
2536 TIter next(fPackages);
2537 TObject *package;
2538 while ((package=next())) {
1ac846e9 2539 // Skip packages already enabled
2540 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2541 TString spkg = package->GetName();
2542 spkg.ReplaceAll(".par", "");
2543 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2544 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2545 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2546 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2547 return kFALSE;
2548 }
2549 } else {
2550 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2551 return kFALSE;
2552 }
2553 }
2554 }
2555 // Do we need to load analysis source files ?
2556 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2557 if (fAnalysisSource.Length()) {
2558 TObjArray *list = fAnalysisSource.Tokenize(" ");
2559 TIter next(list);
2560 TObjString *str;
2561 while((str=(TObjString*)next())) {
2562 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2563 }
2564 if (list) delete list;
2565 }
830acc4c 2566 if (testMode) {
2567 // Register dataset to proof lite.
2568 if (fFileForTestMode.IsNull()) {
2569 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2570 return kFALSE;
2571 }
2572 if (gSystem->AccessPathName(fFileForTestMode)) {
2573 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2574 return kFALSE;
2575 }
2576 TFileCollection *coll = new TFileCollection();
2577 coll->AddFromFile(fFileForTestMode);
8ddad121 2578 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2579 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2580 }
3bdcb562 2581 return kTRUE;
2582 }
c57f56b7 2583
43da816a 2584 // Check if output files have to be taken from the analysis manager
2585 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2586 // Add output files and AOD files
2587 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2588 // Add extra files registered to the analysis manager
f790bc1b 2589 TString extra = GetListOfFiles("ext");
2590 if (!extra.IsNull()) {
149d288c 2591 extra.ReplaceAll(".root", "*.root");
f790bc1b 2592 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2593 fOutputFiles += extra;
84fcd93f 2594 }
149d288c 2595 // Compose the output archive.
b83f84f0 2596 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2597 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2598 }
f7b1cbc2 2599// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2600 if (TestBit(AliAnalysisGrid::kOffline)) {
2601 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2602 \n there nor any job run. You can revise the JDL and analysis \
2603 \n macro then run the same in \"submit\" mode.");
2604 } else if (TestBit(AliAnalysisGrid::kTest)) {
2605 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2606 \n dataset.");
2607 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2608 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2609 \n space and job submitted.");
2610 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2611 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2612 if (fMergeViaJDL) CheckInputData();
5513444a 2613 return kTRUE;
c57f56b7 2614 } else {
2615 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2616 }
2617
348be253 2618 Print();
c57f56b7 2619 if (!Connect()) {
2620 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2621 return kFALSE;
16a4353c 2622 }
d3b18c4c 2623 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2624 if (!CheckInputData()) {
2625 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2626 return kFALSE;
c57f56b7 2627 }
d3339be3 2628 if (!CreateDataset(fDataPattern)) {
2629 TString serror;
2630 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2631 if (fRunNumbers.Length()) serror = "run numbers";
2632 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2633 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2634 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2635 return kFALSE;
2636 }
c57f56b7 2637 WriteAnalysisFile();
2638 WriteAnalysisMacro();
2639 WriteExecutable();
2640 WriteValidationScript();
0f389141 2641 if (fMergeViaJDL) {
2642 WriteMergingMacro();
2643 WriteMergeExecutable();
2644 WriteValidationScript(kTRUE);
2645 }
5513444a 2646 if (!CreateJDL()) return kFALSE;
2647 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2648 if (testMode) {
c57f56b7 2649 // Locally testing the analysis
2650 Info("StartAnalysis", "\n_______________________________________________________________________ \
2651 \n Running analysis script in a daughter shell as on a worker node \
2652 \n_______________________________________________________________________");
149d288c 2653 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2654 TIter next(list);
2655 TObjString *str;
a2f5fc01 2656 TString outputFile;
c57f56b7 2657 while((str=(TObjString*)next())) {
a2f5fc01 2658 outputFile = str->GetString();
2659 Int_t index = outputFile.Index("@");
2660 if (index > 0) outputFile.Remove(index);
2661 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2662 }
2663 delete list;
2664 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2665 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2666// gSystem->Exec("cat stdout");
5513444a 2667 return kFALSE;
c57f56b7 2668 }
5513444a 2669 // Check if submitting is managed by LPM manager
a3e84053 2670 if (fProductionMode) {
5513444a 2671 TString prodfile = fJDLName;
2672 prodfile.ReplaceAll(".jdl", ".prod");
2673 WriteProductionFile(prodfile);
2674 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2675 return kFALSE;
2676 }
a8739e8a 2677 // Submit AliEn job(s)
d2a409b2 2678 gGrid->Cd(fGridOutputDir);
a8739e8a 2679 TGridResult *res;
c57f56b7 2680 TString jobID = "";
d2a409b2 2681 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2682 // Submit a given xml or a set of runs
a8739e8a 2683 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2684 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2685 if (res) {
2686 const char *cjobId = res->GetKey(0,"jobId");
2687 if (!cjobId) {
a03be957 2688 gGrid->Stdout();
2689 gGrid->Stderr();
a8739e8a 2690 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2691 return kFALSE;
a8739e8a 2692 } else {
2693 Info("StartAnalysis", "\n_______________________________________________________________________ \
2694 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2695 \n_______________________________________________________________________",
2696 fJDLName.Data(), cjobId);
2697 jobID = cjobId;
2698 }
2699 delete res;
a03be957 2700 } else {
2701 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2702 return kFALSE;
a8739e8a 2703 }
2704 } else {
d2a409b2 2705 // Submit for a range of enumeration of runs.
a03be957 2706 if (!Submit()) return kFALSE;
c57f56b7 2707 }
a8739e8a 2708
c57f56b7 2709 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2710 \n You may exit at any time and terminate the job later using the option <terminate> \
2711 \n ##################################################################################", jobID.Data());
bb885a9e 2712 gSystem->Exec("aliensh");
5513444a 2713 return kTRUE;
c57f56b7 2714}
2715
f790bc1b 2716//______________________________________________________________________________
2717const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2718{
2719// Get a comma-separated list of output files of the requested type.
2720// Type can be (case unsensitive):
2721// aod - list of aod files (std, extensions and filters)
2722// out - list of output files connected to containers (but not aod's or extras)
2723// ext - list of extra files registered to the manager
2724// ter - list of files produced in terminate
2725 static TString files;
2726 files = "";
2727 TString stype = type;
2728 stype.ToLower();
2729 TString aodfiles, extra;
2730 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2731 if (!mgr) {
2732 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2733 return files.Data();
2734 }
2735 if (mgr->GetOutputEventHandler()) {
2736 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2737 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2738 if (!extraaod.IsNull()) {
2739 aodfiles += ",";
2740 aodfiles += extraaod;
2741 }
2742 }
2743 if (stype.Contains("aod")) {
2744 files = aodfiles;
2745 if (stype == "aod") return files.Data();
2746 }
2747 // Add output files that are not in the list of AOD files
2748 TString outputfiles = "";
2749 TIter next(mgr->GetOutputs());
2750 AliAnalysisDataContainer *output;
2751 const char *filename = 0;
2752 while ((output=(AliAnalysisDataContainer*)next())) {
2753 filename = output->GetFileName();
2754 if (!(strcmp(filename, "default"))) continue;
2755 if (outputfiles.Contains(filename)) continue;
2756 if (aodfiles.Contains(filename)) continue;
2757 if (!outputfiles.IsNull()) outputfiles += ",";
2758 outputfiles += filename;
2759 }
2760 if (stype.Contains("out")) {
2761 if (!files.IsNull()) files += ",";
2762 files += outputfiles;
2763 if (stype == "out") return files.Data();
2764 }
2765 // Add extra files registered to the analysis manager
2766 TString sextra;
2767 extra = mgr->GetExtraFiles();
2768 if (!extra.IsNull()) {
2769 extra.Strip();
2770 extra.ReplaceAll(" ", ",");
2771 TObjArray *fextra = extra.Tokenize(",");
2772 TIter nextx(fextra);
2773 TObject *obj;
2774 while ((obj=nextx())) {
2775 if (aodfiles.Contains(obj->GetName())) continue;
2776 if (outputfiles.Contains(obj->GetName())) continue;
2777 if (sextra.Contains(obj->GetName())) continue;
2778 if (!sextra.IsNull()) sextra += ",";
2779 sextra += obj->GetName();
2780 }
2781 delete fextra;
2782 if (stype.Contains("ext")) {
2783 if (!files.IsNull()) files += ",";
2784 files += sextra;
2785 }
2786 }
2787 if (stype == "ext") return files.Data();
2788 TString termfiles;
2789 if (!fTerminateFiles.IsNull()) {
2790 fTerminateFiles.Strip();
2791 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2792 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2793 TIter nextx(fextra);
2794 TObject *obj;
2795 while ((obj=nextx())) {
2796 if (aodfiles.Contains(obj->GetName())) continue;
2797 if (outputfiles.Contains(obj->GetName())) continue;
2798 if (termfiles.Contains(obj->GetName())) continue;
2799 if (sextra.Contains(obj->GetName())) continue;
2800 if (!termfiles.IsNull()) termfiles += ",";
2801 termfiles += obj->GetName();
2802 }
2803 delete fextra;
2804 }
2805 if (stype.Contains("ter")) {
2806 if (!files.IsNull() && !termfiles.IsNull()) {
2807 files += ",";
2808 files += termfiles;
2809 }
2810 }
2811 return files.Data();
2812}
2813
d2a409b2 2814//______________________________________________________________________________
a03be957 2815Bool_t AliAnalysisAlien::Submit()
d2a409b2 2816{
2817// Submit all master jobs.
2818 Int_t nmasterjobs = fInputFiles->GetEntries();
2819 Long_t tshoot = gSystem->Now();
a03be957 2820 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2821 while (fNsubmitted < nmasterjobs) {
2822 Long_t now = gSystem->Now();
2823 if ((now-tshoot)>30000) {
2824 tshoot = now;
a03be957 2825 if (!SubmitNext()) return kFALSE;
d2a409b2 2826 }
2827 }
a03be957 2828 return kTRUE;
d2a409b2 2829}
2830
0f389141 2831//______________________________________________________________________________
a03be957 2832Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2833{
2834// Submit all merging jobs.
2835 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2836 gGrid->Cd(fGridOutputDir);
2837 TString mergeJDLName = fExecutable;
2838 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2839 if (!fInputFiles) {
2840 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2841 return kFALSE;
2842 }
0f389141 2843 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2844 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2845 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2846 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2847 if (fOutputToRunNo) {
2848 // The output directory is the run number
2849 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2850 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2851 } else {
17b4e9a4 2852 if (!fRunNumbers.Length() && !fRunRange[0]) {
2853 // The output directory is the grid outdir
2854 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2855 runOutDir = fGridOutputDir;
2856 } else {
2857 // The output directory is the master number in 3 digits format
2858 printf("### Submitting merging job for master <%03d>\n", i);
2859 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2860 }
7c2cd90a 2861 }
2862 // Check now the number of merging stages.
2b222f7c 2863 TObjArray *list = fOutputFiles.Tokenize(",");
2864 TIter next(list);
2865 TObjString *str;
2866 TString outputFile;
2867 while((str=(TObjString*)next())) {
2868 outputFile = str->GetString();
2869 Int_t index = outputFile.Index("@");
2870 if (index > 0) outputFile.Remove(index);
2871 if (!fMergeExcludes.Contains(outputFile)) break;
2872 }
2873 delete list;
b3e07543 2874 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2875 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 2876 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 2877 }
a03be957 2878 if (!ntosubmit) return kTRUE;
9f020952 2879 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2880 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2881 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2882 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2883 \n ################################################################################################################");
0f389141 2884 gSystem->Exec("aliensh");
a03be957 2885 return kTRUE;
0f389141 2886}
2887
d2a409b2 2888//______________________________________________________________________________
a03be957 2889Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2890{
08d5b699 2891// Submit next bunch of master jobs if the queue is free. The first master job is
2892// submitted right away, while the next will not be unless the previous was split.
2893// The plugin will not submit new master jobs if there are more that 500 jobs in
2894// waiting phase.
d2a409b2 2895 static Bool_t iscalled = kFALSE;
2896 static Int_t firstmaster = 0;
2897 static Int_t lastmaster = 0;
2898 static Int_t npermaster = 0;
a03be957 2899 if (iscalled) return kTRUE;
d2a409b2 2900 iscalled = kTRUE;
2901 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2902 Int_t ntosubmit = 0;
2903 TGridResult *res;
2904 TString jobID = "";
3bdcb562 2905 Int_t nmasterjobs = fInputFiles->GetEntries();
2906 if (!fNsubmitted) {
2907 ntosubmit = 1;
2908 if (!IsUseSubmitPolicy()) {
90d50a8c 2909 if (nmasterjobs>5)
3bdcb562 2910 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2911 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2912 ntosubmit = nmasterjobs;
2913 }
2914 } else {
d2a409b2 2915 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2916 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2917 // If last master not split, just return
a03be957 2918 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2919 // No more than 100 waiting jobs
08d5b699 2920 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2921 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2922 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2923 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2924 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2925 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2926 }
d2a409b2 2927 for (Int_t i=0; i<ntosubmit; i++) {
2928 // Submit for a range of enumeration of runs.
a03be957 2929 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2930 TString query;
cd11251e 2931 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2932 runOutDir.ReplaceAll(".xml", "");
2933 if (fOutputToRunNo)
2934 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2935 else
2936 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2937 printf("********* %s\n",query.Data());
d2a409b2 2938 res = gGrid->Command(query);
2939 if (res) {
98ca124f 2940 TString cjobId1 = res->GetKey(0,"jobId");
2941 if (!cjobId1.Length()) {
d2a409b2 2942 iscalled = kFALSE;
a03be957 2943 gGrid->Stdout();
2944 gGrid->Stderr();
2945 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2946 return kFALSE;
d2a409b2 2947 } else {
2948 Info("StartAnalysis", "\n_______________________________________________________________________ \
2949 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2950 \n_______________________________________________________________________",
98ca124f 2951 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2952 jobID += cjobId1;
2953 jobID += " ";
98ca124f 2954 lastmaster = cjobId1.Atoi();
d2a409b2 2955 if (!firstmaster) firstmaster = lastmaster;
2956 fNsubmitted++;
2957 }
2958 delete res;
a03be957 2959 } else {
2960 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2961 return kFALSE;
d2a409b2 2962 }
2963 }
2964 iscalled = kFALSE;
a03be957 2965 return kTRUE;
d2a409b2 2966}
2967
c57f56b7 2968//______________________________________________________________________________
2969void AliAnalysisAlien::WriteAnalysisFile()
2970{
f10e8481 2971// Write current analysis manager into the file <analysisFile>
2972 TString analysisFile = fExecutable;
2973 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2974 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2975 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2976 if (!mgr || !mgr->IsInitialized()) {
2977 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2978 return;
2979 }
2980 // Check analysis type
2981 TObject *handler;
2982 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2983 handler = (TObject*)mgr->GetInputEventHandler();
2984 if (handler) {
65eb22f8 2985 if (handler->InheritsFrom("AliMultiInputEventHandler")) {
2986 AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
2987 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2988 if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2989 } else {
2990 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2991 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2992 }
c57f56b7 2993 }
2994 TDirectory *cdir = gDirectory;
f10e8481 2995 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2996 if (file) {
e16a394c 2997 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2998 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2999 // Unless merging makes no sense
3000 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 3001 mgr->Write();
3002 delete file;
fe2d7fc2 3003 // Enable termination for local jobs
3004 mgr->SetSkipTerminate(kFALSE);
c57f56b7 3005 }
3006 if (cdir) cdir->cd();
f10e8481 3007 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3008 }
3009 Bool_t copy = kTRUE;
d3b18c4c 3010 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3011 if (copy) {
3012 CdWork();
3013 TString workdir = gGrid->GetHomeDirectory();
3014 workdir += fGridWorkingDir;
d3b18c4c 3015 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3016 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3017 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3018 }
3019}
3020
3021//______________________________________________________________________________
3022void AliAnalysisAlien::WriteAnalysisMacro()
3023{
3024// Write the analysis macro that will steer the analysis in grid mode.
3025 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3026 ofstream out;
3027 out.open(fAnalysisMacro.Data(), ios::out);
3028 if (!out.good()) {
3029 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3030 return;
3031 }
5b273635 3032 Bool_t hasSTEERBase = kFALSE;
3033 Bool_t hasESD = kFALSE;
3034 Bool_t hasAOD = kFALSE;
3035 Bool_t hasANALYSIS = kFALSE;
3036 Bool_t hasANALYSISalice = kFALSE;
3037 Bool_t hasCORRFW = kFALSE;
c57f56b7 3038 TString func = fAnalysisMacro;
3039 TString type = "ESD";
3040 TString comment = "// Analysis using ";
c85cfc0f 3041 if (IsUseMCchain()) {
3042 type = "MC";
3043 comment += "MC";
3044 } else {
3045 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3046 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3047 type = "AOD";
3048 comment += "AOD";
3049 }
3050 }
0df6ccf2 3051 if (type!="AOD" && fFriendChainName!="") {
3052 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3053 return;
3054 }
c57f56b7 3055 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3056 else comment += " data";
3057 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3058 func.ReplaceAll(".C", "");
3059 out << "void " << func.Data() << "()" << endl;
3060 out << "{" << endl;
3061 out << comment.Data() << endl;
3062 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3063 out << " TStopwatch timer;" << endl;
3064 out << " timer.Start();" << endl << endl;
13f0c00b 3065 // Change temp directory to current one
3066 out << "// Set temporary merging directory to current one" << endl;
3067 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3068 out << "// Set temporary compilation directory to current one" << endl;
3069 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3070 // Reset existing include path
3071 out << "// Reset existing include path and add current directory first in the search" << endl;
3072 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3073 if (!fExecutableCommand.Contains("aliroot")) {
3074 out << "// load base root libraries" << endl;
3075 out << " gSystem->Load(\"libTree\");" << endl;
3076 out << " gSystem->Load(\"libGeom\");" << endl;
3077 out << " gSystem->Load(\"libVMC\");" << endl;
3078 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3079 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3080 }
d5c6455a 3081 if (fAdditionalRootLibs.Length()) {
47a4137d 3082 // in principle libtree /lib geom libvmc etc. can go into this list, too
3083 out << "// Add aditional libraries" << endl;
3084 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3085 TIter next(list);
3086 TObjString *str;
3087 while((str=(TObjString*)next())) {
3088 if (str->GetString().Contains(".so"))
3089 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3090 }
47a4137d 3091 if (list) delete list;
d5c6455a 3092 }
57377eb5 3093 out << "// Load analysis framework libraries" << endl;
652561d5 3094 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3095 if (!fPackages) {
f790bc1b 3096 if (!fExecutableCommand.Contains("aliroot")) {
3097 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3098 out << " gSystem->Load(\"libESD\");" << endl;
3099 out << " gSystem->Load(\"libAOD\");" << endl;
3100 }
4e5c5506 3101 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3102 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3103 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3104 } else {
4e5c5506 3105 TIter next(fPackages);
3106 TObject *obj;
57377eb5 3107 TString pkgname;
57377eb5 3108 while ((obj=next())) {
3109 pkgname = obj->GetName();
4478e6f1 3110 if (pkgname == "STEERBase" ||
3111 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3112 if (pkgname == "ESD" ||
3113 pkgname == "ESD.par") hasESD = kTRUE;
3114 if (pkgname == "AOD" ||
3115 pkgname == "AOD.par") hasAOD = kTRUE;
3116 if (pkgname == "ANALYSIS" ||
3117 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3118 if (pkgname == "ANALYSISalice" ||
3119 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3120 if (pkgname == "CORRFW" ||
3121 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3122 }
3123 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3124 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3125 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3126 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3127 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3128 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3129 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3130 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3131 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3132 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3133 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3134 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3135 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3136 out << "// Compile other par packages" << endl;
3137 next.Reset();
fcc9bb6f 3138 while ((obj=next())) {
3139 pkgname = obj->GetName();
4478e6f1 3140 if (pkgname == "STEERBase" ||
3141 pkgname == "STEERBase.par" ||
3142 pkgname == "ESD" ||
3143 pkgname == "ESD.par" ||
3144 pkgname == "AOD" ||
3145 pkgname == "AOD.par" ||
3146 pkgname == "ANALYSIS" ||
3147 pkgname == "ANALYSIS.par" ||
3148 pkgname == "ANALYSISalice" ||
3149 pkgname == "ANALYSISalice.par" ||
3150 pkgname == "CORRFW" ||
3151 pkgname == "CORRFW.par") continue;
5b273635 3152 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3153 }
4e5c5506 3154 }
b3e07543 3155 out << "// include path" << endl;
3156 // Get the include path from the interpreter and remove entries pointing to AliRoot
3157 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3158 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3159 out << " TIter nextpath(listpaths);" << endl;
3160 out << " TObjString *pname;" << endl;
3161 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3162 out << " TString current = pname->GetName();" << endl;
3163 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3164 out << " gSystem->AddIncludePath(current);" << endl;
3165 out << " }" << endl;
3166 out << " if (listpaths) delete listpaths;" << endl;
3167 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3168 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3169 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3170 if (fAdditionalLibs.Length()) {
3171 out << "// Add aditional AliRoot libraries" << endl;
3172 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3173 TIter next(list);
3174 TObjString *str;
3175 while((str=(TObjString*)next())) {
3176 if (str->GetString().Contains(".so"))
3177 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3178 if (str->GetString().Contains(".par"))
3179 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3180 }
3181 if (list) delete list;
3182 }
3183 out << endl;
c57f56b7 3184 out << "// analysis source to be compiled at runtime (if any)" << endl;
3185 if (fAnalysisSource.Length()) {
3186 TObjArray *list = fAnalysisSource.Tokenize(" ");
3187 TIter next(list);
3188 TObjString *str;
3189 while((str=(TObjString*)next())) {
3190 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3191 }
3192 if (list) delete list;
3193 }
3194 out << endl;
b3e07543 3195// out << " printf(\"Currently load libraries:\\n\");" << endl;
3196// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3197 if (fFastReadOption) {
d3339be3 3198 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3199 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3200 out << "// fast xrootd reading enabled" << endl;
3201 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3202 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3203 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3204 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3205 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3206 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3207 }
c57f56b7 3208 out << "// connect to AliEn and make the chain" << endl;
3209 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3210 out << "// read the analysis manager from file" << endl;
f10e8481 3211 TString analysisFile = fExecutable;
3212 analysisFile.ReplaceAll(".sh", ".root");
3213 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3214 out << " if (!file) return;" << endl;
3215 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3216 out << " AliAnalysisManager *mgr = 0;" << endl;
3217 out << " TKey *key;" << endl;
3218 out << " while ((key=(TKey*)nextkey())) {" << endl;
3219 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3220 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3221 out << " };" << endl;
3222 out << " if (!mgr) {" << endl;
205b201f 3223 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3224 out << " return;" << endl;
3225 out << " }" << endl << endl;
3226 out << " mgr->PrintStatus();" << endl;
52b6a92b 3227 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3228 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3229 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3230 } else {
9a8288ff 3231 if (TestBit(AliAnalysisGrid::kTest))
3232 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3233 else
3234 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3235 }
3236 }
242accb2 3237 if (IsUsingTags()) {
3238 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3239 } else {
3240 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3241 }
c57f56b7 3242 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3243 out << " timer.Stop();" << endl;
3244 out << " timer.Print();" << endl;
c57f56b7 3245 out << "}" << endl << endl;
3246 if (IsUsingTags()) {
3247 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3248 out << "{" << endl;
3249 out << "// Create a chain using tags from the xml file." << endl;
3250 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3251 out << " if (!coll) {" << endl;
3252 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3253 out << " return NULL;" << endl;
3254 out << " }" << endl;
3255 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3256 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3257 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3258 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3259 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3260 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3261 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3262 out << " // Check if the cuts configuration file was provided" << endl;
3263 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3264 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3265 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3266 out << " }" << endl;
0df6ccf2 3267 if (fFriendChainName=="") {
3268 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3269 } else {
3270 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3271 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3272 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3273 }
c57f56b7 3274 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3275 out << " chain->ls();" << endl;
3276 out << " return chain;" << endl;
fcc9bb6f 3277 out << "}" << endl << endl;
c57f56b7 3278 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3279 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3280 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3281 msg += " AliLHCTagCuts *lhcCuts,\n";
3282 msg += " AliDetectorTagCuts *detCuts,\n";
3283 msg += " AliEventTagCuts *evCuts)";
23329835 3284 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3285 }
0df6ccf2 3286 }
3287 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3288 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3289 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3290 out << "{" << endl;
3291 out << "// Create a chain using url's from xml file" << endl;
242accb2 3292 out << " TString filename;" << endl;
3293 out << " Int_t run = 0;" << endl;
c85cfc0f 3294 if (IsUseMCchain()) {
3295 out << " TString treename = \"TE\";" << endl;
3296 } else {
3297 out << " TString treename = type;" << endl;
3298 out << " treename.ToLower();" << endl;
3299 out << " treename += \"Tree\";" << endl;
3300 }
e02fee64 3301 out << " printf(\"***************************************\\n\");" << endl;
3302 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3303 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3304 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3305 out << " if (!coll) {" << endl;
3306 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3307 out << " return NULL;" << endl;
3308 out << " }" << endl;
242accb2 3309 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3310 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3311 if(fFriendChainName!="") {
3312 out << " TChain *chainFriend = new TChain(treename);" << endl;
3313 }
c57f56b7 3314 out << " coll->Reset();" << endl;
0df6ccf2 3315 out << " while (coll->Next()) {" << endl;
242accb2 3316 out << " filename = coll->GetTURL("");" << endl;
3317 out << " if (mgr) {" << endl;
3318 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3319 out << " if (nrun && nrun != run) {" << endl;
3320 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3321 out << " mgr->SetRunFromPath(nrun);" << endl;
3322 out << " run = nrun;" << endl;
3323 out << " }" << endl;
3324 out << " }" << endl;
3325 out << " chain->Add(filename);" << endl;
0df6ccf2 3326 if(fFriendChainName!="") {
3327 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3328 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3329 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3330 out << " chainFriend->Add(fileFriend.Data());" << endl;
3331 }
3332 out << " }" << endl;
c57f56b7 3333 out << " if (!chain->GetNtrees()) {" << endl;
3334 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3335 out << " return NULL;" << endl;
3336 out << " }" << endl;
0df6ccf2 3337 if(fFriendChainName!="") {
3338 out << " chain->AddFriend(chainFriend);" << endl;
3339 }
c57f56b7 3340 out << " return chain;" << endl;
fcc9bb6f 3341 out << "}" << endl << endl;
c57f56b7 3342 }
5b273635 3343 if (hasANALYSISalice) {
3344 out <<"//________________________________________________________________________________" << endl;
3345 out << "Bool_t SetupPar(const char *package) {" << endl;
3346 out << "// Compile the package and set it up." << endl;
3347 out << " TString pkgdir = package;" << endl;
3348 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3349 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3350 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3351 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3352 out << " // Check for BUILD.sh and execute" << endl;
3353 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3354 out << " printf(\"*******************************\\n\");" << endl;
3355 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3356 out << " printf(\"*******************************\\n\");" << endl;
3357 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3358 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3359 out << " gSystem->ChangeDirectory(cdir);" << endl;
3360 out << " return kFALSE;" << endl;
3361 out << " }" << endl;
3362 out << " } else {" << endl;
3363 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3364 out << " gSystem->ChangeDirectory(cdir);" << endl;
3365 out << " return kFALSE;" << endl;
3366 out << " }" << endl;
3367 out << " // Check for SETUP.C and execute" << endl;
3368 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3369 out << " printf(\"*******************************\\n\");" << endl;
3370 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3371 out << " printf(\"*******************************\\n\");" << endl;
3372 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3373 out << " } else {" << endl;
3374 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3375 out << " gSystem->ChangeDirectory(cdir);" << endl;
3376 out << " return kFALSE;" << endl;
3377 out << " }" << endl;
3378 out << " // Restore original workdir" << endl;
3379 out << " gSystem->ChangeDirectory(cdir);" << endl;
3380 out << " return kTRUE;" << endl;
3381 out << "}" << endl;
3382 }
c57f56b7 3383 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3384 }
3385 Bool_t copy = kTRUE;
d3b18c4c 3386 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3387 if (copy) {
3388 CdWork();
3389 TString workdir = gGrid->GetHomeDirectory();
3390 workdir += fGridWorkingDir;
3391 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3392 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3393 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3394 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3395 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3396 }
3397 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3398 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3399 }
3400}
3401
0f389141 3402//______________________________________________________________________________
3403void AliAnalysisAlien::WriteMergingMacro()
3404{
3405// Write a macro to merge the outputs per master job.
3406 if (!fMergeViaJDL) return;
3407 if (!fOutputFiles.Length()) {
3408 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3409 return;
3410 }
3411 TString mergingMacro = fExecutable;
3412 mergingMacro.ReplaceAll(".sh","_merge.C");
3413 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3414 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3415 ofstream out;
3416 out.open(mergingMacro.Data(), ios::out);
3417 if (!out.good()) {
3418 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3419 return;
3420 }
5b273635 3421 Bool_t hasSTEERBase = kFALSE;
3422 Bool_t hasESD = kFALSE;
3423 Bool_t hasAOD = kFALSE;
3424 Bool_t hasANALYSIS = kFALSE;
3425 Bool_t hasANALYSISalice = kFALSE;
3426 Bool_t hasCORRFW = kFALSE;
0f389141 3427 TString func = mergingMacro;
3428 TString comment;
3429 func.ReplaceAll(".C", "");
f47d5cb4 3430 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3431 out << "{" << endl;
3432 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3433 out << " TStopwatch timer;" << endl;
3434 out << " timer.Start();" << endl << endl;
b3e07543 3435 // Reset existing include path
3436 out << "// Reset existing include path and add current directory first in the search" << endl;
3437 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3438 if (!fExecutableCommand.Contains("aliroot")) {
3439 out << "// load base root libraries" << endl;
3440 out << " gSystem->Load(\"libTree\");" << endl;
3441 out << " gSystem->Load(\"libGeom\");" << endl;
3442 out << " gSystem->Load(\"libVMC\");" << endl;
3443 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3444 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3445 }
0f389141 3446 if (fAdditionalRootLibs.Length()) {
3447 // in principle libtree /lib geom libvmc etc. can go into this list, too
3448 out << "// Add aditional libraries" << endl;
3449 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3450 TIter next(list);
3451 TObjString *str;
3452 while((str=(TObjString*)next())) {
3453 if (str->GetString().Contains(".so"))
3454 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3455 }
3456 if (list) delete list;
3457 }
0f389141 3458 out << "// Load analysis framework libraries" << endl;
3459 if (!fPackages) {
7c2cd90a 3460 if (!fExecutableCommand.Contains("aliroot")) {
3461 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3462 out << " gSystem->Load(\"libESD\");" << endl;
3463 out << " gSystem->Load(\"libAOD\");" << endl;
3464 }
0f389141 3465 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3466 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3467 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3468 } else {
3469 TIter next(fPackages);
3470 TObject *obj;
3471 TString pkgname;
5b273635 3472 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3473 while ((obj=next())) {
3474 pkgname = obj->GetName();
3475 if (pkgname == "STEERBase" ||
3476 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3477 if (pkgname == "ESD" ||
3478 pkgname == "ESD.par") hasESD = kTRUE;
3479 if (pkgname == "AOD" ||
3480 pkgname == "AOD.par") hasAOD = kTRUE;
3481 if (pkgname == "ANALYSIS" ||
3482 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3483 if (pkgname == "ANALYSISalice" ||
3484 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3485 if (pkgname == "CORRFW" ||
3486 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3487 }
5b273635 3488 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3489 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3490 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3491 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3492 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3493 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3494 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3495 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3496 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3497 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3498 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3499 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3500 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3501 out << "// Compile other par packages" << endl;
3502 next.Reset();
3503 while ((obj=next())) {
3504 pkgname = obj->GetName();
3505 if (pkgname == "STEERBase" ||
3506 pkgname == "STEERBase.par" ||
3507 pkgname == "ESD" ||
3508 pkgname == "ESD.par" ||
3509 pkgname == "AOD" ||
3510 pkgname == "AOD.par" ||
3511 pkgname == "ANALYSIS" ||
3512 pkgname == "ANALYSIS.par" ||
3513 pkgname == "ANALYSISalice" ||
3514 pkgname == "ANALYSISalice.par" ||
3515 pkgname == "CORRFW" ||
3516 pkgname == "CORRFW.par") continue;
5b273635 3517 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3518 }
3519 }
b3e07543 3520 out << "// include path" << endl;
3521 // Get the include path from the interpreter and remove entries pointing to AliRoot
3522 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3523 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3524 out << " TIter nextpath(listpaths);" << endl;
3525 out << " TObjString *pname;" << endl;
3526 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3527 out << " TString current = pname->GetName();" << endl;
3528 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3529 out << " gSystem->AddIncludePath(current);" << endl;
3530 out << " }" << endl;
3531 out << " if (listpaths) delete listpaths;" << endl;
3532 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3533 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3534 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3535 if (fAdditionalLibs.Length()) {
3536 out << "// Add aditional AliRoot libraries" << endl;
3537 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3538 TIter next(list);
3539 TObjString *str;
3540 while((str=(TObjString*)next())) {
3541 if (str->GetString().Contains(".so"))
3542 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3543 }
3544 if (list) delete list;
3545 }
3546 out << endl;
3547 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3548 if (fAnalysisSource.Length()) {
3549 TObjArray *list = fAnalysisSource.Tokenize(" ");
3550 TIter next(list);
3551 TObjString *str;
3552 while((str=(TObjString*)next())) {
3553 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3554 }
3555 if (list) delete list;
3556 }
149d288c 3557 out << endl;
3558
0f389141 3559 if (fFastReadOption) {
3560 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3561 out << "// fast xrootd reading enabled" << endl;
3562 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3563 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3564 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3565 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3566 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3567 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3568 }
3569 // Change temp directory to current one
3570 out << "// Set temporary merging directory to current one" << endl;
3571 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3572 out << "// Set temporary compilation directory to current one" << endl;
3573 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 3574 out << "// Connect to AliEn" << endl;
3575 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3576 out << " TString outputDir = dir;" << endl;
f790bc1b 3577 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3578 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3579 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3580 out << " TIter *iter = new TIter(list);" << endl;
3581 out << " TObjString *str;" << endl;
a2f5fc01 3582 out << " TString outputFile;" << endl;
0f389141 3583 out << " Bool_t merged = kTRUE;" << endl;
3584 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3585 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3586 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3587 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3588 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3589 out << " // Skip already merged outputs" << endl;
a2f5fc01 3590 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3591 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3592 out << " continue;" << endl;
3593 out << " }" << endl;
a2f5fc01 3594 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3595 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3596 out << " if (!merged) {" << endl;
a2f5fc01 3597 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3598 out << " return;" << endl;
0f389141 3599 out << " }" << endl;
3600 out << " }" << endl;
7c2cd90a 3601 out << " // all outputs merged, validate" << endl;
3602 out << " ofstream out;" << endl;
3603 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3604 out << " out.close();" << endl;
3605 out << " // read the analysis manager from file" << endl;
f866cba5 3606 TString analysisFile = fExecutable;
3607 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3608 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3609 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3610 out << " if (!file) return;" << endl;
f866cba5 3611 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3612 out << " AliAnalysisManager *mgr = 0;" << endl;
3613 out << " TKey *key;" << endl;
3614 out << " while ((key=(TKey*)nextkey())) {" << endl;
3615 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3616 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3617 out << " };" << endl;
3618 out << " if (!mgr) {" << endl;
3619 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3620 out << " return;" << endl;
3621 out << " }" << endl << endl;
5c4250fc 3622 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3623 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3624 out << " mgr->PrintStatus();" << endl;
3625 if (AliAnalysisManager::GetAnalysisManager()) {
3626 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3627 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3628 } else {
9a8288ff 3629 if (TestBit(AliAnalysisGrid::kTest))
3630 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3631 else
3632 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3633 }
3634 }
9a8288ff 3635 out << " TTree *tree = NULL;" << endl;
3636 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3637 out << "}" << endl << endl;
5b273635 3638 if (hasANALYSISalice) {
3639 out <<"//________________________________________________________________________________" << endl;
3640 out << "Bool_t SetupPar(const char *package) {" << endl;
3641 out << "// Compile the package and set it up." << endl;
3642 out << " TString pkgdir = package;" << endl;
3643 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3644 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3645 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3646 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3647 out << " // Check for BUILD.sh and execute" << endl;
3648 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3649 out << " printf(\"*******************************\\n\");" << endl;
3650 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3651 out << " printf(\"*******************************\\n\");" << endl;
3652 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3653 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3654 out << " gSystem->ChangeDirectory(cdir);" << endl;
3655 out << " return kFALSE;" << endl;
3656 out << " }" << endl;
3657 out << " } else {" << endl;
3658 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3659 out << " gSystem->ChangeDirectory(cdir);" << endl;
3660 out << " return kFALSE;" << endl;
3661 out << " }" << endl;
3662 out << " // Check for SETUP.C and execute" << endl;
3663 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3664 out << " printf(\"*******************************\\n\");" << endl;
3665 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3666 out << " printf(\"*******************************\\n\");" << endl;
3667 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3668 out << " } else {" << endl;
3669 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3670 out << " gSystem->ChangeDirectory(cdir);" << endl;
3671 out << " return kFALSE;" << endl;
3672 out << " }" << endl;
3673 out << " // Restore original workdir" << endl;
3674 out << " gSystem->ChangeDirectory(cdir);" << endl;
3675 out << " return kTRUE;" << endl;
3676 out << "}" << endl;
3677 }
0f389141 3678 }
3679 Bool_t copy = kTRUE;
d3b18c4c 3680 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3681 if (copy) {
3682 CdWork();
3683 TString workdir = gGrid->GetHomeDirectory();
3684 workdir += fGridWorkingDir;
3685 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3686 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3687 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3688 }
3689}
3690
3691//______________________________________________________________________________
3692Bool_t AliAnalysisAlien::SetupPar(const char *package)
3693{
205b201f 3694// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3695// Note that for loading the compiled library. The current directory should have precedence in
3696// LD_LIBRARY_PATH
3697 TString pkgdir = package;
3698 pkgdir.ReplaceAll(".par","");
b50a41c3 3699 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3700 TString cdir = gSystem->WorkingDirectory();
3701 gSystem->ChangeDirectory(pkgdir);
3702 // Check for BUILD.sh and execute
3703 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3704 printf("**************************************************\n");
3705 printf("*** Building PAR archive %s\n", package);
3706 printf("**************************************************\n");
3707 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3708 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3709 gSystem->ChangeDirectory(cdir);
3710 return kFALSE;
3711 }
3712 } else {
3713 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3714 gSystem->ChangeDirectory(cdir);
3715 return kFALSE;
3716 }
3717 // Check for SETUP.C and execute
3718 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3719 printf("**************************************************\n");
3720 printf("*** Setup PAR archive %s\n", package);
3721 printf("**************************************************\n");
3722 gROOT->Macro("PROOF-INF/SETUP.C");
3723 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3724 } else {
3725 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3726 gSystem->ChangeDirectory(cdir);
3727 return kFALSE;
3728 }
3729 // Restore original workdir
3730 gSystem->ChangeDirectory(cdir);
3731 return kTRUE;
3732}
3733
c57f56b7 3734//______________________________________________________________________________
3735void AliAnalysisAlien::WriteExecutable()
3736{
3737// Generate the alien executable script.
3738 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3739 ofstream out;
3740 out.open(fExecutable.Data(), ios::out);
3741 if (out.bad()) {
5513444a 3742 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3743 return;
3744 }
3745 out << "#!/bin/bash" << endl;
2a4d5166 3746 // Make sure we can properly compile par files
3747 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3748 out << "echo \"=========================================\"" << endl;
3749 out << "echo \"############## PATH : ##############\"" << endl;
3750 out << "echo $PATH" << endl;
3751 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3752 out << "echo $LD_LIBRARY_PATH" << endl;
3753 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3754 out << "echo $ROOTSYS" << endl;
3755 out << "echo \"############## which root : ##############\"" << endl;
3756 out << "which root" << endl;
3757 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3758 out << "echo $ALICE_ROOT" << endl;
3759 out << "echo \"############## which aliroot : ##############\"" << endl;
3760 out << "which aliroot" << endl;
9c5ddadc 3761 out << "echo \"############## system limits : ##############\"" << endl;
3762 out << "ulimit -a" << endl;
3763 out << "echo \"############## memory : ##############\"" << endl;
3764 out << "free -m" << endl;
c57f56b7 3765 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3766 out << fExecutableCommand << " ";
631c0b05 3767 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3768 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3769 out << "echo \"############## memory after: ##############\"" << endl;
3770 out << "free -m" << endl;
c57f56b7 3771 }
3772 Bool_t copy = kTRUE;
d3b18c4c 3773 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3774 if (copy) {
3775 CdWork();
3776 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3777 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3778 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3779 workdir += fGridWorkingDir;
3780 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3781 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3782 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3783 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3784 }
3785}
3786
0f389141 3787//______________________________________________________________________________
3788void AliAnalysisAlien::WriteMergeExecutable()
3789{
3790// Generate the alien executable script for the merging job.
3791 if (!fMergeViaJDL) return;
3792 TString mergeExec = fExecutable;
3793 mergeExec.ReplaceAll(".sh", "_merge.sh");
3794 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3795 ofstream out;
3796 out.open(mergeExec.Data(), ios::out);
3797 if (out.bad()) {
3798 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3799 return;
3800 }
3801 out << "#!/bin/bash" << endl;
2a4d5166 3802 // Make sure we can properly compile par files
3803 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3804 out << "echo \"=========================================\"" << endl;
3805 out << "echo \"############## PATH : ##############\"" << endl;
3806 out << "echo $PATH" << endl;
3807 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3808 out << "echo $LD_LIBRARY_PATH" << endl;
3809 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3810 out << "echo $ROOTSYS" << endl;
3811 out << "echo \"############## which root : ##############\"" << endl;
3812 out << "which root" << endl;
3813 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3814 out << "echo $ALICE_ROOT" << endl;
3815 out << "echo \"############## which aliroot : ##############\"" << endl;
3816 out << "which aliroot" << endl;
3817 out << "echo \"############## system limits : ##############\"" << endl;
3818 out << "ulimit -a" << endl;
3819 out << "echo \"############## memory : ##############\"" << endl;
3820 out << "free -m" << endl;
3821 out << "echo \"=========================================\"" << endl << endl;
0f389141 3822 TString mergeMacro = fExecutable;
3823 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3824 if (IsOneStageMerging())
3825 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3826 else
f47d5cb4 3827 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3828 out << fExecutableCommand << " " << "$ARG" << endl;
3829 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3830 out << "echo \"############## memory after: ##############\"" << endl;
3831 out << "free -m" << endl;
0f389141 3832 }
3833 Bool_t copy = kTRUE;
d3b18c4c 3834 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3835 if (copy) {
3836 CdWork();
3837 TString workdir = gGrid->GetHomeDirectory();
3838 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3839 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3840 workdir += fGridWorkingDir;
3841 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3842 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3843 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3844 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3845 }
3846}
3847
c57f56b7 3848//______________________________________________________________________________
5513444a 3849void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3850{
3851// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3852// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3853// Next lines: full_path_to_dataset XXX (XXX is a string)
3854// To submit, one has to: submit jdl XXX for all lines
3855 ofstream out;
3856 out.open(filename, ios::out);
3857 if (out.bad()) {
3858 Error("WriteProductionFile", "Bad file name: %s", filename);
3859 return;
3860 }
d3b18c4c 3861 TString workdir;
3862 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3863 workdir = gGrid->GetHomeDirectory();
5513444a 3864 workdir += fGridWorkingDir;
f5e8c702 3865 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3866 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3867 out << locjdl << " " << njobspermaster << endl;
5513444a 3868 Int_t nmasterjobs = fInputFiles->GetEntries();
3869 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3870 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3871 runOutDir.ReplaceAll(".xml", "");
3872 if (fOutputToRunNo)
3873 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3874 else
3875 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3876 }
d3b18c4c 3877 if (gGrid) {
3878 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3879 if (FileExists(filename)) gGrid->Rm(filename);
3880 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3881 }
5513444a 3882}
3883
3884//______________________________________________________________________________
0f389141 3885void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3886{
3887// Generate the alien validation script.
3888 // Generate the validation script
3889 TObjString *os;
d3b18c4c 3890 if (fValidationScript.IsNull()) {
3891 fValidationScript = fExecutable;
3892 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3893 }
3894 TString validationScript = fValidationScript;
3895 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3896 if (!Connect()) {
3897 Error("WriteValidationScript", "Alien connection required");
3898 return;
3899 }
d3b18c4c 3900 if (!fTerminateFiles.IsNull()) {
3901 fTerminateFiles.Strip();
3902 fTerminateFiles.ReplaceAll(" ",",");
3903 }
a2f5fc01 3904 TString outStream = "";
3905 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3906 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3907 ofstream out;
0d5d317c 3908 out.open(validationScript, ios::out);
c57f56b7 3909 out << "#!/bin/bash" << endl;
3910 out << "##################################################" << endl;
3911 out << "validateout=`dirname $0`" << endl;
3912 out << "validatetime=`date`" << endl;
3913 out << "validated=\"0\";" << endl;
3914 out << "error=0" << endl;
3915 out << "if [ -z $validateout ]" << endl;
3916 out << "then" << endl;
3917 out << " validateout=\".\"" << endl;
3918 out << "fi" << endl << endl;
3919 out << "cd $validateout;" << endl;
3920 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3921 out << "echo \"*******************************************************\"" << outStream << endl;
3922 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3923 out << "" << endl;
a2f5fc01 3924 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3925 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3926 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3927 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3928 out << "ls -la ./" << outStream << endl;
3929 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3930 out << "##################################################" << endl;
ebec370a 3931 out << "" << endl;
3932
3933 out << "if [ ! -f stderr ] ; then" << endl;
3934 out << " error=1" << endl;
a2f5fc01 3935 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3936 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3937 out << "fi" << endl;
3938
b34c9f51 3939 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3940 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3941 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3942 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3943 out << "" << endl;
3944
ebec370a 3945 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3946 out << " error=1" << endl;
a2f5fc01 3947 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3948 out << " echo \"$parArch\" " << outStream << endl;
3949 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3950 out << "fi" << endl;
3951
3952 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3953 out << " error=1" << endl;
a2f5fc01 3954 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3955 out << " echo \"$segViol\" " << outStream << endl;
3956 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3957 out << "fi" << endl;
3958
3959 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3960 out << " error=1" << endl;
a2f5fc01 3961 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3962 out << " echo \"$segFault\" " << outStream << endl;
3963 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3964 out << "fi" << endl;
3965
b34c9f51 3966 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3967 out << " error=1" << endl;
a2f5fc01 3968 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3969 out << " echo \"$glibcErr\" " << outStream << endl;
3970 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3971 out << "fi" << endl;
3972
ebec370a 3973 // Part dedicated to the specific analyses running into the train
3974
d3b18c4c 3975 TString outputFiles = fOutputFiles;
3976 if (merge && !fTerminateFiles.IsNull()) {
3977 outputFiles += ",";
3978 outputFiles += fTerminateFiles;
3979 }
3980 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3981 TIter next1(arr);
a2f5fc01 3982 TString outputFile;
bbd88db8 3983 while (!merge && (os=(TObjString*)next1())) {
3984 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3985 outputFile = os->GetString();
3986 Int_t index = outputFile.Index("@");
3987 if (index > 0) outputFile.Remove(index);
bbd88db8 3988 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3989 if (outputFile.Contains("*")) continue;
3990 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3991 out << " error=1" << endl;
7c2cd90a 3992 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3993 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3994 out << "fi" << endl;
3995 }
3996 delete arr;
7c2cd90a 3997 out << "if ! [ -f outputs_valid ] ; then" << endl;
3998 out << " error=1" << endl;
3999 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
4000 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
4001 out << "fi" << endl;
923e2ca5 4002
c57f56b7 4003 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 4004 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 4005 if (!IsKeepLogs()) {
4006 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 4007 outStream = "";
149d288c 4008 out << " rm -f std*" << endl;
4009 }
c57f56b7 4010 out << "fi" << endl;
4011
a2f5fc01 4012 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4013 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4014 out << "cd -" << endl;
4015 out << "exit $error" << endl;
4016 }
4017 Bool_t copy = kTRUE;
d3b18c4c 4018 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4019 if (copy) {
4020 CdWork();
4021 TString workdir = gGrid->GetHomeDirectory();
4022 workdir += fGridWorkingDir;
d3b18c4c 4023 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4024 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4025 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4026 }
4027}