]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Setting compilaton directory to current one
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
b3e07543 61 fMaxMergeStages(0),
d2a409b2 62 fNsubmitted(0),
a3e84053 63 fProductionMode(0),
cd11251e 64 fOutputToRunNo(0),
0f389141 65 fMergeViaJDL(0),
66 fFastReadOption(0),
e1c22e21 67 fOverwriteMode(1),
149d288c 68 fNreplicas(2),
3bdcb562 69 fNproofWorkers(0),
27734f0e 70 fNproofWorkersPerSlave(0),
3bdcb562 71 fProofReset(0),
c57f56b7 72 fRunNumbers(),
73 fExecutable(),
0a1c1f7f 74 fExecutableCommand(),
c57f56b7 75 fArguments(),
631c0b05 76 fExecutableArgs(),
c57f56b7 77 fAnalysisMacro(),
78 fAnalysisSource(),
d3b18c4c 79 fValidationScript(),
d5c6455a 80 fAdditionalRootLibs(),
c57f56b7 81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
648174cf 86 fExternalPackages(),
c57f56b7 87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
e7c71df0 95 fDatasetName(),
c57f56b7 96 fJDLName(),
d3b18c4c 97 fTerminateFiles(),
bb885a9e 98 fMergeExcludes(),
f965131e 99 fIncludePath(),
bb885a9e 100 fCloseSE(),
0df6ccf2 101 fFriendChainName(),
c6cb3634 102 fJobTag(),
648174cf 103 fOutputSingle(),
5fce53f4 104 fRunPrefix(),
3bdcb562 105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
f47d5cb4 110 fMergeDirName(),
4e5c5506 111 fInputFiles(0),
112 fPackages(0)
c57f56b7 113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
0f389141 122 fMergingJDL(NULL),
c57f56b7 123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
bb885a9e 128 fNtestFiles(0),
319593fb 129 fNrunsPerMaster(0),
16a4353c 130 fMaxMergeFiles(0),
b3e07543 131 fMaxMergeStages(0),
d2a409b2 132 fNsubmitted(0),
a3e84053 133 fProductionMode(0),
cd11251e 134 fOutputToRunNo(0),
0f389141 135 fMergeViaJDL(0),
136 fFastReadOption(0),
e1c22e21 137 fOverwriteMode(1),
149d288c 138 fNreplicas(2),
3bdcb562 139 fNproofWorkers(0),
27734f0e 140 fNproofWorkersPerSlave(0),
3bdcb562 141 fProofReset(0),
c57f56b7 142 fRunNumbers(),
143 fExecutable(),
0a1c1f7f 144 fExecutableCommand(),
c57f56b7 145 fArguments(),
631c0b05 146 fExecutableArgs(),
c57f56b7 147 fAnalysisMacro(),
148 fAnalysisSource(),
d3b18c4c 149 fValidationScript(),
d5c6455a 150 fAdditionalRootLibs(),
c57f56b7 151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
648174cf 156 fExternalPackages(),
c57f56b7 157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
e7c71df0 165 fDatasetName(),
c57f56b7 166 fJDLName(),
d3b18c4c 167 fTerminateFiles(),
bb885a9e 168 fMergeExcludes(),
f965131e 169 fIncludePath(),
bb885a9e 170 fCloseSE(),
0df6ccf2 171 fFriendChainName(),
c6cb3634 172 fJobTag(),
648174cf 173 fOutputSingle(),
5fce53f4 174 fRunPrefix(),
3bdcb562 175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
f47d5cb4 180 fMergeDirName(),
4e5c5506 181 fInputFiles(0),
182 fPackages(0)
c57f56b7 183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
0f389141 192 fMergingJDL(NULL),
c57f56b7 193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 198 fNtestFiles(other.fNtestFiles),
319593fb 199 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 200 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 201 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 202 fNsubmitted(other.fNsubmitted),
a3e84053 203 fProductionMode(other.fProductionMode),
cd11251e 204 fOutputToRunNo(other.fOutputToRunNo),
0f389141 205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
149d288c 208 fNreplicas(other.fNreplicas),
3bdcb562 209 fNproofWorkers(other.fNproofWorkers),
27734f0e 210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 211 fProofReset(other.fProofReset),
c57f56b7 212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
0a1c1f7f 214 fExecutableCommand(other.fExecutableCommand),
c57f56b7 215 fArguments(other.fArguments),
631c0b05 216 fExecutableArgs(other.fExecutableArgs),
c57f56b7 217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 219 fValidationScript(other.fValidationScript),
d5c6455a 220 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
648174cf 226 fExternalPackages(other.fExternalPackages),
c57f56b7 227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
e7c71df0 235 fDatasetName(other.fDatasetName),
c57f56b7 236 fJDLName(other.fJDLName),
d3b18c4c 237 fTerminateFiles(other.fTerminateFiles),
bb885a9e 238 fMergeExcludes(other.fMergeExcludes),
f965131e 239 fIncludePath(other.fIncludePath),
bb885a9e 240 fCloseSE(other.fCloseSE),
0df6ccf2 241 fFriendChainName(other.fFriendChainName),
c6cb3634 242 fJobTag(other.fJobTag),
648174cf 243 fOutputSingle(other.fOutputSingle),
5fce53f4 244 fRunPrefix(other.fRunPrefix),
3bdcb562 245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
f47d5cb4 250 fMergeDirName(other.fMergeDirName),
4e5c5506 251 fInputFiles(0),
252 fPackages(0)
c57f56b7 253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
c57f56b7 259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
4e5c5506 266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
c57f56b7 273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
0f389141 280 if (fMergingJDL) delete fMergingJDL;
c57f56b7 281 if (fInputFiles) delete fInputFiles;
4e5c5506 282 if (fPackages) delete fPackages;
c57f56b7 283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 298 fNtestFiles = other.fNtestFiles;
a3e84053 299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 301 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
cd11251e 304 fOutputToRunNo = other.fOutputToRunNo;
0f389141 305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
149d288c 308 fNreplicas = other.fNreplicas;
3bdcb562 309 fNproofWorkers = other.fNproofWorkers;
27734f0e 310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 311 fProofReset = other.fProofReset;
c57f56b7 312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
0a1c1f7f 314 fExecutableCommand = other.fExecutableCommand;
c57f56b7 315 fArguments = other.fArguments;
631c0b05 316 fExecutableArgs = other.fExecutableArgs;
c57f56b7 317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 319 fValidationScript = other.fValidationScript;
d5c6455a 320 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
648174cf 326 fExternalPackages = other.fExternalPackages;
c57f56b7 327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
e7c71df0 335 fDatasetName = other.fDatasetName;
c57f56b7 336 fJDLName = other.fJDLName;
d3b18c4c 337 fTerminateFiles = other.fTerminateFiles;
bb885a9e 338 fMergeExcludes = other.fMergeExcludes;
f965131e 339 fIncludePath = other.fIncludePath;
bb885a9e 340 fCloseSE = other.fCloseSE;
0df6ccf2 341 fFriendChainName = other.fFriendChainName;
c6cb3634 342 fJobTag = other.fJobTag;
648174cf 343 fOutputSingle = other.fOutputSingle;
5fce53f4 344 fRunPrefix = other.fRunPrefix;
3bdcb562 345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
f47d5cb4 350 fMergeDirName = other.fMergeDirName;
c57f56b7 351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
4e5c5506 358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
c57f56b7 365 }
366 return *this;
367}
368
74013535 369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
f965131e 377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 391 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 392}
393
98690895 394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunList(const char* runList)
396{
397// Add several runs into the list of runs; they are expected to be separated by a blank character.
398 TString sList = runList;
399 TObjArray *list = sList.Tokenize(" ");
400 Int_t n = list->GetEntries();
401 for (Int_t i = 0; i < n; i++) {
402 TObjString *os = (TObjString*)list->At(i);
403 AddRunNumber(os->GetString().Atoi());
404 }
405 delete list;
406}
407
ee75cfc3 408//______________________________________________________________________________
409void AliAnalysisAlien::AddRunNumber(const char* run)
410{
411// Add a run number to the list of runs to be processed.
412 if (fRunNumbers.Length()) fRunNumbers += " ";
413 fRunNumbers += run;
414}
415
c57f56b7 416//______________________________________________________________________________
417void AliAnalysisAlien::AddDataFile(const char *lfn)
418{
419// Adds a data file to the input to be analysed. The file should be a valid LFN
420// or point to an existing file in the alien workdir.
421 if (!fInputFiles) fInputFiles = new TObjArray();
422 fInputFiles->Add(new TObjString(lfn));
423}
648174cf 424
425//______________________________________________________________________________
426void AliAnalysisAlien::AddExternalPackage(const char *package)
427{
428// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
429 if (fExternalPackages) fExternalPackages += " ";
430 fExternalPackages += package;
431}
432
c57f56b7 433//______________________________________________________________________________
434Bool_t AliAnalysisAlien::Connect()
435{
436// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
437 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 438 if (fProductionMode) return kTRUE;
c57f56b7 439 if (!gGrid) {
440 Info("Connect", "Trying to connect to AliEn ...");
441 TGrid::Connect("alien://");
442 }
443 if (!gGrid || !gGrid->IsConnected()) {
444 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
445 return kFALSE;
446 }
447 fUser = gGrid->GetUser();
448 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
449 return kTRUE;
450}
451
452//______________________________________________________________________________
453void AliAnalysisAlien::CdWork()
454{
455// Check validity of alien workspace. Create directory if possible.
456 if (!Connect()) {
457 Error("CdWork", "Alien connection required");
458 return;
459 }
460 TString homedir = gGrid->GetHomeDirectory();
461 TString workdir = homedir + fGridWorkingDir;
923e2ca5 462 if (DirectoryExists(workdir)) {
463 gGrid->Cd(workdir);
464 return;
465 }
466 // Work directory not existing - create it
467 gGrid->Cd(homedir);
b93f8109 468 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 469 gGrid->Cd(fGridWorkingDir);
d3b18c4c 470 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 471 } else {
d3b18c4c 472 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 473 workdir.Data(), homedir.Data());
474 fGridWorkingDir = "";
475 }
c57f56b7 476}
477
348be253 478//______________________________________________________________________________
479Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
480{
481// Check if file copying is possible.
d3b18c4c 482 if (fProductionMode) return kTRUE;
348be253 483 if (!Connect()) {
484 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
485 return kFALSE;
486 }
d3339be3 487 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
488 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 489 // Check if alien_CLOSE_SE is defined
490 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
491 if (!closeSE.IsNull()) {
492 Info("CheckFileCopy", "Your current close storage is pointing to: \
493 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
494 } else {
495 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
496 }
497 // Check if grid directory exists.
498 if (!DirectoryExists(alienpath)) {
499 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
500 return kFALSE;
501 }
d3339be3 502 TFile f("plugin_test_copy", "RECREATE");
348be253 503 // User may not have write permissions to current directory
504 if (f.IsZombie()) {
505 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
506 gSystem->WorkingDirectory());
507 return kFALSE;
508 }
509 f.Close();
d3339be3 510 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
511 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
512 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 513 \n# 1. Make sure you have write permissions there. If this is the case: \
514 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
515 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
516 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
517 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
518 gSystem->Unlink(f.GetName());
519 return kFALSE;
520 }
521 gSystem->Unlink(f.GetName());
d3339be3 522 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
523 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 524 return kTRUE;
525}
526
c57f56b7 527//______________________________________________________________________________
528Bool_t AliAnalysisAlien::CheckInputData()
529{
530// Check validity of input data. If necessary, create xml files.
d3b18c4c 531 if (fProductionMode) return kTRUE;
a8739e8a 532 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 533 if (!fGridDataDir.Length()) {
534 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
535 return kFALSE;
536 }
9f020952 537 if (fMergeViaJDL) {
538 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
539 return kFALSE;
540 }
d2a409b2 541 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 542 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
543 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 544 return kTRUE;
c57f56b7 545 }
546 // Process declared files
a2f5fc01 547 Bool_t isCollection = kFALSE;
548 Bool_t isXml = kFALSE;
549 Bool_t useTags = kFALSE;
c57f56b7 550 Bool_t checked = kFALSE;
d3b18c4c 551 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 552 TString file;
553 TString workdir = gGrid->GetHomeDirectory();
554 workdir += fGridWorkingDir;
555 if (fInputFiles) {
556 TObjString *objstr;
557 TIter next(fInputFiles);
558 while ((objstr=(TObjString*)next())) {
559 file = workdir;
560 file += "/";
561 file += objstr->GetString();
562 // Store full lfn path
563 if (FileExists(file)) objstr->SetString(file);
564 else {
565 file = objstr->GetName();
566 if (!FileExists(objstr->GetName())) {
567 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
568 objstr->GetName(), workdir.Data());
569 return kFALSE;
570 }
571 }
572 Bool_t iscoll, isxml, usetags;
573 CheckDataType(file, iscoll, isxml, usetags);
574 if (!checked) {
575 checked = kTRUE;
a2f5fc01 576 isCollection = iscoll;
577 isXml = isxml;
578 useTags = usetags;
579 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 580 } else {
a2f5fc01 581 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 582 Error("CheckInputData", "Some conflict was found in the types of inputs");
583 return kFALSE;
584 }
585 }
586 }
587 }
588 // Process requested run numbers
a8739e8a 589 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 590 // Check validity of alien data directory
591 if (!fGridDataDir.Length()) {
592 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
593 return kFALSE;
594 }
923e2ca5 595 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 596 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
597 return kFALSE;
598 }
a2f5fc01 599 if (isCollection) {
c57f56b7 600 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
601 return kFALSE;
602 }
603
a2f5fc01 604 if (checked && !isXml) {
c57f56b7 605 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
606 return kFALSE;
607 }
608 // Check validity of run number(s)
609 TObjArray *arr;
610 TObjString *os;
74013535 611 TString format;
319593fb 612 Int_t nruns = 0;
904f9f5f 613 TString schunk, schunk2;
c57f56b7 614 TString path;
615 if (!checked) {
616 checked = kTRUE;
a2f5fc01 617 useTags = fDataPattern.Contains("tag");
618 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 619 }
a2f5fc01 620 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 621 Error("CheckInputData", "Cannot mix input files using/not using tags");
622 return kFALSE;
623 }
624 if (fRunNumbers.Length()) {
a8739e8a 625 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 626 arr = fRunNumbers.Tokenize(" ");
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
629 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 630 if (!DirectoryExists(path)) {
631 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 632 continue;
c57f56b7 633 }
634 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
635 TString msg = "\n##### file: ";
636 msg += path;
637 msg += " type: xml_collection;";
a2f5fc01 638 if (useTags) msg += " using_tags: Yes";
c57f56b7 639 else msg += " using_tags: No";
23329835 640 Info("CheckDataType", "%s", msg.Data());
319593fb 641 if (fNrunsPerMaster<2) {
d2a409b2 642 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 643 } else {
644 nruns++;
645 if (((nruns-1)%fNrunsPerMaster) == 0) {
646 schunk = os->GetString();
647 }
648 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
649 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 650 AddDataFile(schunk);
319593fb 651 }
c57f56b7 652 }
653 delete arr;
a8739e8a 654 } else {
655 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
656 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 657 format = Form("%%s/%s ", fRunPrefix.Data());
658 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 659 if (!DirectoryExists(path)) {
a8739e8a 660 continue;
661 }
74013535 662 format = Form("%%s/%s.xml", fRunPrefix.Data());
663 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 664 TString msg = "\n##### file: ";
665 msg += path;
666 msg += " type: xml_collection;";
a2f5fc01 667 if (useTags) msg += " using_tags: Yes";
a8739e8a 668 else msg += " using_tags: No";
23329835 669 Info("CheckDataType", "%s", msg.Data());
319593fb 670 if (fNrunsPerMaster<2) {
74013535 671 format = Form("%s.xml", fRunPrefix.Data());
672 AddDataFile(Form(format.Data(),irun));
319593fb 673 } else {
674 nruns++;
675 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 676 schunk = Form(fRunPrefix.Data(),irun);
319593fb 677 }
74013535 678 format = Form("_%s.xml", fRunPrefix.Data());
679 schunk2 = Form(format.Data(), irun);
319593fb 680 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 681 schunk += schunk2;
d2a409b2 682 AddDataFile(schunk);
319593fb 683 }
a8739e8a 684 }
904f9f5f 685 if (!fInputFiles) {
686 schunk += schunk2;
687 AddDataFile(schunk);
688 }
c57f56b7 689 }
690 return kTRUE;
691}
692
693//______________________________________________________________________________
694Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
695{
696// Create dataset for the grid data directory + run number.
d3b18c4c 697 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 698 if (!Connect()) {
699 Error("CreateDataset", "Cannot create dataset with no grid connection");
700 return kFALSE;
701 }
702
703 // Cd workspace
d3b18c4c 704 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 705 TString workdir = gGrid->GetHomeDirectory();
706 workdir += fGridWorkingDir;
707
708 // Compose the 'find' command arguments
74013535 709 TString format;
c57f56b7 710 TString command;
711 TString options = "-x collection ";
bb885a9e 712 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 713 TString conditions = "";
714
715 TString file;
716 TString path;
319593fb 717 Int_t nruns = 0;
904f9f5f 718 TString schunk, schunk2;
ab254fd1 719 TGridCollection *cbase=0, *cadd=0;
d2a409b2 720 if (!fRunNumbers.Length() && !fRunRange[0]) {
721 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
722 // Make a single data collection from data directory.
723 path = fGridDataDir;
923e2ca5 724 if (!DirectoryExists(path)) {
d2a409b2 725 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
726 return kFALSE;
727 }
923e2ca5 728// CdWork();
d2a409b2 729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 731 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 732 command = "find ";
733 command += options;
734 command += path;
735 command += " ";
736 command += pattern;
737 command += conditions;
84fcd93f 738 printf("command: %s\n", command.Data());
d2a409b2 739 TGridResult *res = gGrid->Command(command);
740 if (res) delete res;
741 // Write standard output to file
742 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 743 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
744 Bool_t nullFile = kFALSE;
745 if (!hasGrep) {
d3339be3 746 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
747 } else {
a2f5fc01 748 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
749 if (nullFile) {
defd7a3a 750 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
751 return kFALSE;
752 }
d3339be3 753 }
0f389141 754 }
755 Bool_t fileExists = FileExists(file);
756 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 757 // Copy xml file to alien space
0f389141 758 if (fileExists) gGrid->Rm(file);
d2a409b2 759 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
760 if (!FileExists(file)) {
761 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
762 return kFALSE;
763 }
764 // Update list of files to be processed.
765 }
766 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
767 return kTRUE;
768 }
c57f56b7 769 // Several runs
a2f5fc01 770 Bool_t nullResult = kTRUE;
a8739e8a 771 if (fRunNumbers.Length()) {
772 TObjArray *arr = fRunNumbers.Tokenize(" ");
773 TObjString *os;
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 777 if (!DirectoryExists(path)) continue;
778// CdWork();
a8739e8a 779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
780 else file = Form("%s.xml", os->GetString().Data());
319593fb 781 // If local collection file does not exist, create it via 'find' command.
d3339be3 782 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 783 command = "find ";
784 command += options;
785 command += path;
786 command += pattern;
787 command += conditions;
788 TGridResult *res = gGrid->Command(command);
789 if (res) delete res;
790 // Write standard output to file
791 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 792 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
793 Bool_t nullFile = kFALSE;
794 if (!hasGrep) {
d3339be3 795 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
796 } else {
a2f5fc01 797 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
798 if (nullFile) {
defd7a3a 799 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
800 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
801 continue;
802 }
d3339be3 803 }
a2f5fc01 804 nullResult = kFALSE;
d3339be3 805 }
a8739e8a 806 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 807 // Check if there is one run per master job.
808 if (fNrunsPerMaster<2) {
809 if (FileExists(file)) {
0f389141 810 if (fOverwriteMode) gGrid->Rm(file);
811 else {
812 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
813 continue;
814 }
319593fb 815 }
816 // Copy xml file to alien space
817 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
818 if (!FileExists(file)) {
819 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
820 delete arr;
821 return kFALSE;
822 }
823 } else {
824 nruns++;
825 if (((nruns-1)%fNrunsPerMaster) == 0) {
826 schunk = os->GetString();
827 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
828 } else {
829 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 830 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 831 cbase->Add(cadd);
832 delete cadd;
833 }
834 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
835 continue;
836 }
837 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 838 if (FileExists(schunk)) {
839 if (fOverwriteMode) gGrid->Rm(file);
840 else {
841 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
842 continue;
843 }
319593fb 844 }
84fcd93f 845 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 846 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 847 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 848 if (!FileExists(schunk)) {
849 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
850 delete arr;
851 return kFALSE;
852 }
d3339be3 853 }
a8739e8a 854 }
855 delete arr;
a2f5fc01 856 if (nullResult) {
d3339be3 857 Error("CreateDataset", "No valid dataset corresponding to the query!");
858 return kFALSE;
859 }
a8739e8a 860 } else {
861 // Process a full run range.
862 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 863 format = Form("%%s/%s ", fRunPrefix.Data());
864 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 865 if (!DirectoryExists(path)) continue;
866// CdWork();
74013535 867 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 868 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 869 else file = Form(format.Data(), irun);
0f389141 870 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
871 if (fOverwriteMode) gGrid->Rm(file);
872 else {
873 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
874 continue;
875 }
a8739e8a 876 }
319593fb 877 // If local collection file does not exist, create it via 'find' command.
d3339be3 878 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 879 command = "find ";
880 command += options;
881 command += path;
882 command += pattern;
883 command += conditions;
884 TGridResult *res = gGrid->Command(command);
885 if (res) delete res;
886 // Write standard output to file
887 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 888 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
889 Bool_t nullFile = kFALSE;
890 if (!hasGrep) {
d3339be3 891 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
892 } else {
a2f5fc01 893 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
894 if (nullFile) {
defd7a3a 895 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
896 continue;
897 }
d3339be3 898 }
a2f5fc01 899 nullResult = kFALSE;
319593fb 900 }
a8739e8a 901 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 902 // Check if there is one run per master job.
903 if (fNrunsPerMaster<2) {
904 if (FileExists(file)) {
0f389141 905 if (fOverwriteMode) gGrid->Rm(file);
906 else {
907 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
908 continue;
909 }
319593fb 910 }
911 // Copy xml file to alien space
912 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
913 if (!FileExists(file)) {
914 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
915 return kFALSE;
916 }
917 } else {
918 nruns++;
95e5b448 919 // Check if the collection for the chunk exist locally.
920 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 921 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
922 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
923 else continue;
924 }
84fcd93f 925 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 926 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 927 schunk = Form(fRunPrefix.Data(), irun);
319593fb 928 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
929 } else {
930 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 cbase->Add(cadd);
932 delete cadd;
933 }
74013535 934 format = Form("%%s_%s.xml", fRunPrefix.Data());
935 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 936 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 937 continue;
938 }
904f9f5f 939 schunk = schunk2;
319593fb 940 if (FileExists(schunk)) {
0f389141 941 if (fOverwriteMode) gGrid->Rm(schunk);
942 else {
943 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
944 continue;
945 }
319593fb 946 }
84fcd93f 947 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 948 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 949 if (FileExists(schunk)) {
0f389141 950 if (fOverwriteMode) gGrid->Rm(schunk);
951 else {
952 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
953 continue;
954 }
95e5b448 955 }
319593fb 956 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
957 if (!FileExists(schunk)) {
958 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
959 return kFALSE;
960 }
961 }
c57f56b7 962 }
a2f5fc01 963 if (nullResult) {
d3339be3 964 Error("CreateDataset", "No valid dataset corresponding to the query!");
965 return kFALSE;
966 }
a8739e8a 967 }
c57f56b7 968 return kTRUE;
969}
970
971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CreateJDL()
973{
974// Generate a JDL file according to current settings. The name of the file is
975// specified by fJDLName.
976 Bool_t error = kFALSE;
977 TObjArray *arr = 0;
978 Bool_t copy = kTRUE;
d3b18c4c 979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 980 Bool_t generate = kTRUE;
981 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
982 if (!Connect()) {
983 Error("CreateJDL", "Alien connection required");
984 return kFALSE;
985 }
986 // Check validity of alien workspace
d3b18c4c 987 TString workdir;
988 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
989 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 990 workdir += fGridWorkingDir;
991 if (generate) {
992 TObjString *os;
993 if (!fInputFiles) {
994 Error("CreateJDL()", "Define some input files for your analysis.");
995 error = kTRUE;
996 }
997 // Compose list of input files
998 // Check if output files were defined
999 if (!fOutputFiles.Length()) {
1000 Error("CreateJDL", "You must define at least one output file");
1001 error = kTRUE;
1002 }
1003 // Check if an output directory was defined and valid
1004 if (!fGridOutputDir.Length()) {
1005 Error("CreateJDL", "You must define AliEn output directory");
1006 error = kTRUE;
1007 } else {
d3b18c4c 1008 if (!fProductionMode) {
1009 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1010 if (!DirectoryExists(fGridOutputDir)) {
1011 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1012 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1013 } else {
1014 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1015 // error = kTRUE;
1016 }
67a3b254 1017 } else {
1018 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1019 }
1020 gGrid->Cd(workdir);
1021 }
c57f56b7 1022 }
1023 // Exit if any error up to now
1024 if (error) return kFALSE;
1025 // Set JDL fields
0f389141 1026 if (!fUser.IsNull()) {
1027 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1028 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1029 }
1030 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1031 TString mergeExec = fExecutable;
1032 mergeExec.ReplaceAll(".sh", "_merge.sh");
1033 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1034 mergeExec.ReplaceAll(".sh", ".C");
1035 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1036 if (!fArguments.IsNull())
1037 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1038 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1039 else {
1040 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1041 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1042 }
b3e07543 1043
1f0d1ca2 1044 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1045 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1046 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048
0f389141 1049 if (fMaxInitFailed > 0) {
c57f56b7 1050 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1051 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1052 }
1053 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1054 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1055 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1056 }
1057 if (!IsOneStageMerging()) {
f47d5cb4 1058 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1059 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1060 }
1061 if (fSplitMode.Length()) {
c57f56b7 1062 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1063 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1064 }
1065 fMergingJDL->SetValue("Split", "\"se\"");
1066 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1067 if (!fAliROOTVersion.IsNull()) {
1068 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1069 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1070 }
1071 if (!fROOTVersion.IsNull()) {
c57f56b7 1072 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1073 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1074 }
1075 if (!fAPIVersion.IsNull()) {
c57f56b7 1076 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1077 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1078 }
648174cf 1079 if (!fExternalPackages.IsNull()) {
1080 arr = fExternalPackages.Tokenize(" ");
1081 TIter next(arr);
1082 while ((os=(TObjString*)next())) {
1083 TString pkgname = os->GetString();
1084 Int_t index = pkgname.Index("::");
1085 TString pkgversion = pkgname(index+2, pkgname.Length());
1086 pkgname.Remove(index);
1087 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1088 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1089 }
1090 delete arr;
1091 }
0f389141 1092 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1093 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1094 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1096 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1097 TString analysisFile = fExecutable;
1098 analysisFile.ReplaceAll(".sh", ".root");
1099 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1100 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1101 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1102 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1103 if (fAdditionalLibs.Length()) {
1104 arr = fAdditionalLibs.Tokenize(" ");
1105 TIter next(arr);
1106 while ((os=(TObjString*)next())) {
1107 if (os->GetString().Contains(".so")) continue;
1108 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1109 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1110 }
1111 delete arr;
1112 }
4e5c5506 1113 if (fPackages) {
1114 TIter next(fPackages);
1115 TObject *obj;
0f389141 1116 while ((obj=next())) {
4e5c5506 1117 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1118 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1119 }
4e5c5506 1120 }
c57f56b7 1121 if (fOutputArchive.Length()) {
1122 arr = fOutputArchive.Tokenize(" ");
1123 TIter next(arr);
0f389141 1124 Bool_t first = kTRUE;
1125 const char *comment = "Files to be archived";
1126 const char *comment1 = comment;
1127 while ((os=(TObjString*)next())) {
1128 if (!first) comment = NULL;
1129 if (!os->GetString().Contains("@") && fCloseSE.Length())
1130 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1131 else
1132 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1133 first = kFALSE;
1134 }
c57f56b7 1135 delete arr;
f790bc1b 1136 // Output archive for the merging jdl
1137 TString outputArchive;
1138 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1139 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1140 // Add normal output files, extra files + terminate files
1141 TString files = GetListOfFiles("outextter");
1142 // Do not register merge excludes
1143 if (!fMergeExcludes.IsNull()) {
1144 arr = fMergeExcludes.Tokenize(" ");
1145 TIter next1(arr);
1146 while ((os=(TObjString*)next1())) {
1147 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1148 files.ReplaceAll(os->GetString(),"");
1149 }
66d71516 1150 delete arr;
0f389141 1151 }
f790bc1b 1152 files.ReplaceAll(".root", "*.root");
b83f84f0 1153 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1154 } else {
95f84779 1155 TString files = fOutputArchive;
1156 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1157 outputArchive = files;
f790bc1b 1158 }
0f389141 1159 arr = outputArchive.Tokenize(" ");
705adb3e 1160 TIter next2(arr);
0f389141 1161 comment = comment1;
1162 first = kTRUE;
705adb3e 1163 while ((os=(TObjString*)next2())) {
0f389141 1164 if (!first) comment = NULL;
0cdf65a8 1165 TString currentfile = os->GetString();
0cdf65a8 1166 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1167 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1168 else
0cdf65a8 1169 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1170 first = kFALSE;
1171 }
1172 delete arr;
c57f56b7 1173 }
149d288c 1174 arr = fOutputFiles.Tokenize(",");
c57f56b7 1175 TIter next(arr);
0f389141 1176 Bool_t first = kTRUE;
f790bc1b 1177 const char *comment = "Files to be saved";
43da816a 1178 while ((os=(TObjString*)next())) {
e1eaf596 1179 // Ignore ouputs in jdl that are also in outputarchive
1180 TString sout = os->GetString();
f790bc1b 1181 sout.ReplaceAll("*", "");
1182 sout.ReplaceAll(".root", "");
e1eaf596 1183 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1184 if (fOutputArchive.Contains(sout)) continue;
0f389141 1185 if (!first) comment = NULL;
43da816a 1186 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1187 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1188 else
0f389141 1189 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1190 first = kFALSE;
f790bc1b 1191 if (fMergeExcludes.Contains(sout)) continue;
1192 if (!os->GetString().Contains("@") && fCloseSE.Length())
1193 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1194 else
1195 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1196 }
c57f56b7 1197 delete arr;
0f389141 1198 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1199 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1200 TString validationScript = fValidationScript;
0f389141 1201 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1202 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1203 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1204 if (fMasterResubmitThreshold) {
1205 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1206 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1207 }
d2a409b2 1208 // Write a jdl with 2 input parameters: collection name and output dir name.
1209 WriteJDL(copy);
c57f56b7 1210 }
1211 // Copy jdl to grid workspace
a8739e8a 1212 if (copy) {
b5e4aaa7 1213 // Check if an output directory was defined and valid
1214 if (!fGridOutputDir.Length()) {
1215 Error("CreateJDL", "You must define AliEn output directory");
1216 return kFALSE;
1217 } else {
1218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1219 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1220 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1221 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1222 } else {
1223 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 }
1227 gGrid->Cd(workdir);
1228 }
648174cf 1229 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1230 TString mergeJDLName = fExecutable;
1231 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1232 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1233 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1234 if (fProductionMode) {
648174cf 1235 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1236 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1237 }
648174cf 1238 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1239 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1240 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1241 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1242 if (fMergeViaJDL) {
1243 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1244 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1245 }
648174cf 1246 }
c57f56b7 1247 if (fAdditionalLibs.Length()) {
e7c71df0 1248 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
c57f56b7 1252 if (os->GetString().Contains(".so")) continue;
4e5c5506 1253 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1254 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1255 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1256 }
1257 delete arr;
1258 }
4e5c5506 1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 while ((obj=next())) {
fdbbc7be 1263 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1265 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1266 }
1267 }
c57f56b7 1268 }
1269 return kTRUE;
1270}
1271
a8739e8a 1272//______________________________________________________________________________
d2a409b2 1273Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1274{
1275// Writes one or more JDL's corresponding to findex. If findex is negative,
1276// all run numbers are considered in one go (jdl). For non-negative indices
1277// they correspond to the indices in the array fInputFiles.
1278 if (!fInputFiles) return kFALSE;
d3b18c4c 1279 TObject *os;
1280 TString workdir;
1281 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1282 workdir += fGridWorkingDir;
f47d5cb4 1283 TString stageName = "$2";
1284 if (fProductionMode) stageName = "$4";
1285 if (!fMergeDirName.IsNull()) {
1286 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1287 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1288 } else {
1289 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1290 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1291 }
d3b18c4c 1292 if (fProductionMode) {
a8739e8a 1293 TIter next(fInputFiles);
b3e07543 1294 while ((os=next())) {
d3b18c4c 1295 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1296 }
d3b18c4c 1297 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1298 } else {
1299 if (!fRunNumbers.Length() && !fRunRange[0]) {
1300 // One jdl with no parameters in case input data is specified by name.
1301 TIter next(fInputFiles);
1302 while ((os=next()))
1303 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1304 if (!fOutputSingle.IsNull())
1305 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1306 else {
1307 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1308 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1309 }
1310 } else {
1311 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1312 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1313 if (!fOutputSingle.IsNull()) {
1314 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1315 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1316 } else {
1317 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1318 }
1319 }
a8739e8a 1320 }
1321
a8739e8a 1322 // Generate the JDL as a string
1323 TString sjdl = fGridJDL->Generate();
0f389141 1324 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1325 // Final merge jdl
1326 if (!fMergeDirName.IsNull()) {
1327 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1328 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1329 } else {
1330 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1331 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1332 }
b3e07543 1333 TString sjdl2 = fMergingJDL->Generate();
1334 Int_t index, index1;
a8739e8a 1335 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1336 sjdl.ReplaceAll("(member", "\n (member");
1337 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1338 sjdl.ReplaceAll("{", "{\n ");
1339 sjdl.ReplaceAll("};", "\n};");
1340 sjdl.ReplaceAll("{\n \n", "{\n");
1341 sjdl.ReplaceAll("\n\n", "\n");
1342 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1343 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1344 sjdl1.ReplaceAll("(member", "\n (member");
1345 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1346 sjdl1.ReplaceAll("{", "{\n ");
1347 sjdl1.ReplaceAll("};", "\n};");
1348 sjdl1.ReplaceAll("{\n \n", "{\n");
1349 sjdl1.ReplaceAll("\n\n", "\n");
1350 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1351 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1352 sjdl2.ReplaceAll("(member", "\n (member");
1353 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1354 sjdl2.ReplaceAll("{", "{\n ");
1355 sjdl2.ReplaceAll("};", "\n};");
1356 sjdl2.ReplaceAll("{\n \n", "{\n");
1357 sjdl2.ReplaceAll("\n\n", "\n");
1358 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1359 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1360 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1361 index = sjdl.Index("JDLVariables");
1362 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1363 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1364 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1365 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1366 index = fJobTag.Index(":");
1367 if (index < 0) index = fJobTag.Length();
1368 TString jobTag = fJobTag;
f47d5cb4 1369 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1370 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1371 if (fProductionMode) {
1372 sjdl1.Prepend("# Generated merging jdl (production mode) \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = train number \
1375 \n# $3 = production (like LHC10b) \
1376 \n# $4 = merging stage \
1377 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1378 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1379 sjdl2.Prepend("# Generated merging jdl \
1380 \n# $1 = full alien path to output directory to be merged \
1381 \n# $2 = train number \
1382 \n# $3 = production (like LHC10b) \
1383 \n# $4 = merging stage \
1384 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1385 } else {
1386 sjdl1.Prepend("# Generated merging jdl \
1387 \n# $1 = full alien path to output directory to be merged \
1388 \n# $2 = merging stage \
03335324 1389 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1390 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1391 sjdl2.Prepend("# Generated merging jdl \
1392 \n# $1 = full alien path to output directory to be merged \
1393 \n# $2 = merging stage \
03335324 1394 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1395 }
0f389141 1396 index = sjdl1.Index("JDLVariables");
1397 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1398 index = sjdl2.Index("JDLVariables");
1399 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1400 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1401 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1402 index = sjdl2.Index("Split =");
1403 if (index>=0) {
1404 index1 = sjdl2.Index("\n", index);
1405 sjdl2.Remove(index, index1-index+1);
1406 }
1407 index = sjdl2.Index("SplitMaxInputFileNumber");
1408 if (index>=0) {
1409 index1 = sjdl2.Index("\n", index);
1410 sjdl2.Remove(index, index1-index+1);
1411 }
1412 index = sjdl2.Index("InputDataCollection");
1413 if (index>=0) {
1414 index1 = sjdl2.Index(";", index);
1415 sjdl2.Remove(index, index1-index+1);
1416 }
1417 index = sjdl2.Index("InputDataListFormat");
1418 if (index>=0) {
1419 index1 = sjdl2.Index("\n", index);
1420 sjdl2.Remove(index, index1-index+1);
1421 }
1422 index = sjdl2.Index("InputDataList");
1423 if (index>=0) {
1424 index1 = sjdl2.Index("\n", index);
1425 sjdl2.Remove(index, index1-index+1);
1426 }
f47d5cb4 1427 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1428 // Write jdl to file
a8739e8a 1429 ofstream out;
d2a409b2 1430 out.open(fJDLName.Data(), ios::out);
a8739e8a 1431 if (out.bad()) {
d3b18c4c 1432 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1433 return kFALSE;
1434 }
1435 out << sjdl << endl;
b3e07543 1436 out.close();
0f389141 1437 TString mergeJDLName = fExecutable;
1438 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1439 if (fMergeViaJDL) {
1440 ofstream out1;
1441 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1442 if (out1.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1444 return kFALSE;
1445 }
1446 out1 << sjdl1 << endl;
b3e07543 1447 out1.close();
1448 ofstream out2;
1449 TString finalJDL = mergeJDLName;
1450 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1451 out2.open(finalJDL.Data(), ios::out);
1452 if (out2.bad()) {
1453 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1454 return kFALSE;
1455 }
1456 out2 << sjdl2 << endl;
1457 out2.close();
0f389141 1458 }
a8739e8a 1459
1460 // Copy jdl to grid workspace
1461 if (!copy) {
d3b18c4c 1462 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1463 } else {
d2a409b2 1464 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1465 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1466 TString finalJDL = mergeJDLName;
1467 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1468 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1469 if (fProductionMode) {
b5fe9cba 1470 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1471 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1472 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1473 }
d2a409b2 1474 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1475 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1476 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1477 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1478 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1479 if (fMergeViaJDL) {
b3e07543 1480 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1481 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1482 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1483 }
a8739e8a 1484 }
1485 return kTRUE;
1486}
1487
c57f56b7 1488//______________________________________________________________________________
5513444a 1489Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1490{
1491// Returns true if file exists.
5513444a 1492 if (!gGrid) return kFALSE;
2ea55496 1493 TString slfn = lfn;
1494 slfn.ReplaceAll("alien://","");
1495 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1496 if (!res) return kFALSE;
1497 TMap *map = dynamic_cast<TMap*>(res->At(0));
1498 if (!map) {
1499 delete res;
1500 return kFALSE;
1501 }
1502 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1503 if (!objs || !objs->GetString().Length()) {
1504 delete res;
1505 return kFALSE;
1506 }
1507 delete res;
1508 return kTRUE;
1509}
1510
923e2ca5 1511//______________________________________________________________________________
1512Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1513{
1514// Returns true if directory exists. Can be also a path.
1515 if (!gGrid) return kFALSE;
1516 // Check if dirname is a path
1517 TString dirstripped = dirname;
1518 dirstripped = dirstripped.Strip();
1519 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1520 TString dir = gSystem->BaseName(dirstripped);
1521 dir += "/";
1522 TString path = gSystem->DirName(dirstripped);
1523 TGridResult *res = gGrid->Ls(path, "-F");
1524 if (!res) return kFALSE;
1525 TIter next(res);
1526 TMap *map;
1527 TObject *obj;
1528 while ((map=dynamic_cast<TMap*>(next()))) {
1529 obj = map->GetValue("name");
1530 if (!obj) break;
1531 if (dir == obj->GetName()) {
1532 delete res;
1533 return kTRUE;
1534 }
1535 }
1536 delete res;
1537 return kFALSE;
1538}
1539
c57f56b7 1540//______________________________________________________________________________
a2f5fc01 1541void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1542{
1543// Check input data type.
a2f5fc01 1544 isCollection = kFALSE;
1545 isXml = kFALSE;
1546 useTags = kFALSE;
c57f56b7 1547 if (!gGrid) {
1548 Error("CheckDataType", "No connection to grid");
1549 return;
1550 }
a2f5fc01 1551 isCollection = IsCollection(lfn);
c57f56b7 1552 TString msg = "\n##### file: ";
1553 msg += lfn;
a2f5fc01 1554 if (isCollection) {
c57f56b7 1555 msg += " type: raw_collection;";
1556 // special treatment for collections
a2f5fc01 1557 isXml = kFALSE;
c57f56b7 1558 // check for tag files in the collection
1559 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1560 if (!res) {
1561 msg += " using_tags: No (unknown)";
23329835 1562 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1563 return;
1564 }
1565 const char* typeStr = res->GetKey(0, "origLFN");
1566 if (!typeStr || !strlen(typeStr)) {
1567 msg += " using_tags: No (unknown)";
23329835 1568 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1569 return;
1570 }
1571 TString file = typeStr;
a2f5fc01 1572 useTags = file.Contains(".tag");
1573 if (useTags) msg += " using_tags: Yes";
c57f56b7 1574 else msg += " using_tags: No";
23329835 1575 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1576 return;
1577 }
1578 TString slfn(lfn);
1579 slfn.ToLower();
a2f5fc01 1580 isXml = slfn.Contains(".xml");
1581 if (isXml) {
c57f56b7 1582 // Open xml collection and check if there are tag files inside
1583 msg += " type: xml_collection;";
1584 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1585 if (!coll) {
1586 msg += " using_tags: No (unknown)";
23329835 1587 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1588 return;
1589 }
1590 TMap *map = coll->Next();
1591 if (!map) {
1592 msg += " using_tags: No (unknown)";
23329835 1593 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1594 return;
1595 }
1596 map = (TMap*)map->GetValue("");
1597 TString file;
1598 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1599 useTags = file.Contains(".tag");
c57f56b7 1600 delete coll;
a2f5fc01 1601 if (useTags) msg += " using_tags: Yes";
c57f56b7 1602 else msg += " using_tags: No";
23329835 1603 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1604 return;
1605 }
a2f5fc01 1606 useTags = slfn.Contains(".tag");
c57f56b7 1607 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1608 else msg += " type: unknown file;";
a2f5fc01 1609 if (useTags) msg += " using_tags: Yes";
c57f56b7 1610 else msg += " using_tags: No";
23329835 1611 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1612}
1613
4e5c5506 1614//______________________________________________________________________________
1615void AliAnalysisAlien::EnablePackage(const char *package)
1616{
1617// Enables a par file supposed to exist in the current directory.
1618 TString pkg(package);
1619 pkg.ReplaceAll(".par", "");
1620 pkg += ".par";
1621 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1622 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1623 return;
1624 }
1625 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1626 Info("EnablePackage", "AliEn plugin will use .par packages");
1627 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1628 if (!fPackages) {
1629 fPackages = new TObjArray();
1630 fPackages->SetOwner();
1631 }
1632 fPackages->Add(new TObjString(pkg));
1633}
1634
3bdcb562 1635//______________________________________________________________________________
1636TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1637{
1638// Make a tree from files having the location specified in fFileForTestMode.
1639// Inspired from JF's CreateESDChain.
1640 if (fFileForTestMode.IsNull()) {
1641 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1642 return NULL;
1643 }
1644 if (gSystem->AccessPathName(fFileForTestMode)) {
1645 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1646 return NULL;
1647 }
1648 // Open the file
1649 ifstream in;
1650 in.open(fFileForTestMode);
1651 Int_t count = 0;
1652 // Read the input list of files and add them to the chain
1653 TString line;
1654 TChain *chain = new TChain(treeName);
1655 while (in.good())
1656 {
1657 in >> line;
1658 if (line.IsNull()) continue;
1659 if (count++ == fNtestFiles) break;
1660 TString esdFile(line);
1661 TFile *file = TFile::Open(esdFile);
1662 if (file) {
1663 if (!file->IsZombie()) chain->Add(esdFile);
1664 file->Close();
1665 } else {
1666 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1667 }
1668 }
1669 in.close();
1670 if (!chain->GetListOfFiles()->GetEntries()) {
1671 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1672 delete chain;
1673 return NULL;
1674 }
1675// chain->ls();
1676 return chain;
1677}
1678
d2a409b2 1679//______________________________________________________________________________
1680const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1681{
1682// Get job status for all jobs with jobid>jobidstart.
1683 static char mstatus[20];
1684 mstatus[0] = '\0';
1685 nrunning = 0;
1686 nwaiting = 0;
1687 nerror = 0;
1688 ndone = 0;
1689 TGridJobStatusList *list = gGrid->Ps("");
1690 if (!list) return mstatus;
1691 Int_t nentries = list->GetSize();
1692 TGridJobStatus *status;
1693 Int_t pid;
1694 for (Int_t ijob=0; ijob<nentries; ijob++) {
1695 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1696 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1697 if (pid<jobidstart) continue;
1698 if (pid == lastid) {
8ddad121 1699 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1700 }
1701 switch (status->GetStatus()) {
1702 case TGridJobStatus::kWAITING:
1703 nwaiting++; break;
1704 case TGridJobStatus::kRUNNING:
1705 nrunning++; break;
1706 case TGridJobStatus::kABORTED:
1707 case TGridJobStatus::kFAIL:
1708 case TGridJobStatus::kUNKNOWN:
1709 nerror++; break;
1710 case TGridJobStatus::kDONE:
1711 ndone++;
1712 }
1713 }
1714 list->Delete();
1715 delete list;
1716 return mstatus;
1717}
1718
c57f56b7 1719//______________________________________________________________________________
1720Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1721{
1722// Returns true if file is a collection. Functionality duplicated from
1723// TAlien::Type() because we don't want to directly depend on TAlien.
1724 if (!gGrid) {
1725 Error("IsCollection", "No connection to grid");
1726 return kFALSE;
1727 }
1728 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1729 if (!res) return kFALSE;
1730 const char* typeStr = res->GetKey(0, "type");
1731 if (!typeStr || !strlen(typeStr)) return kFALSE;
1732 if (!strcmp(typeStr, "collection")) return kTRUE;
1733 delete res;
1734 return kFALSE;
1735}
1736
fe2d7fc2 1737//______________________________________________________________________________
1738Bool_t AliAnalysisAlien::IsSingleOutput() const
1739{
1740// Check if single-ouput option is on.
1741 return (!fOutputSingle.IsNull());
1742}
1743
16a4353c 1744//______________________________________________________________________________
1745void AliAnalysisAlien::Print(Option_t *) const
1746{
1747// Print current plugin settings.
84fcd93f 1748 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1750 if (mgr && mgr->IsProofMode()) {
1751 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1752 if (TestBit(AliAnalysisGrid::kTest))
1753 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1754 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1755 if (!fProofDataSet.IsNull())
1756 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1757 if (fProofReset==1)
1758 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1759 if (fProofReset>1)
1760 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (!fRootVersionForProof.IsNull())
1762 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1763 else
1764 printf("= ROOT version requested________________________ default\n");
1765 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1766 if (!fAliRootMode.IsNull())
1767 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1768 if (fNproofWorkers)
1769 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1770 if (fNproofWorkersPerSlave)
1771 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1772 if (TestSpecialBit(kClearPackages))
1773 printf("= ClearPackages requested...\n");
1774 if (fIncludePath.Data())
1775 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1776 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1777 if (fPackages && fPackages->GetEntries()) {
1778 TIter next(fPackages);
1779 TObject *obj;
1780 TString list;
1781 while ((obj=next())) list += obj->GetName();
1782 printf("= Par files to be used: ________________________ %s\n", list.Data());
1783 }
1784 if (TestSpecialBit(kProofConnectGrid))
1785 printf("= Requested PROOF connection to grid\n");
1786 return;
1787 }
e1c22e21 1788 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1789 if (fOverwriteMode) {
1790 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1791 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1792 }
348be253 1793 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1794 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1795 printf("= Production mode:______________________________ %d\n", fProductionMode);
1796 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1797 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1798 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1799 if (fUser.Length())
84fcd93f 1800 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1801 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1802 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1803 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1804 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1805 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1806 if (fRunNumbers.Length())
84fcd93f 1807 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1808 if (fRunRange[0])
74013535 1809 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1810 if (!fRunRange[0] && !fRunNumbers.Length()) {
1811 TIter next(fInputFiles);
1812 TObject *obj;
1813 TString list;
1814 while ((obj=next())) list += obj->GetName();
84fcd93f 1815 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1816 }
1817 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1818 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1819 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1820 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1821 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1822 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1823 printf("=====================================================================\n");
1824 printf("= Job price: ___________________________________ %d\n", fPrice);
1825 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1826 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1827 if (fMaxInitFailed>0)
84fcd93f 1828 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1829 if (fMasterResubmitThreshold>0)
84fcd93f 1830 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1831 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1832 if (fNrunsPerMaster>0)
84fcd93f 1833 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1834 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1835 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1836 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1837 if (fArguments.Length())
84fcd93f 1838 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1839 if (fExecutableArgs.Length())
1840 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1841 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1842 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1843 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1844 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1845 if (fDatasetName)
84fcd93f 1846 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1847 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1848 if (fIncludePath.Data())
84fcd93f 1849 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1850 if (fCloseSE.Length())
84fcd93f 1851 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1852 if (fFriendChainName.Length())
84fcd93f 1853 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1854 if (fPackages && fPackages->GetEntries()) {
16a4353c 1855 TIter next(fPackages);
1856 TObject *obj;
1857 TString list;
1858 while ((obj=next())) list += obj->GetName();
84fcd93f 1859 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1860 }
1861}
1862
c57f56b7 1863//______________________________________________________________________________
1864void AliAnalysisAlien::SetDefaults()
1865{
1866// Set default values for everything. What cannot be filled will be left empty.
1867 if (fGridJDL) delete fGridJDL;
1868 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1869 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1870 fPrice = 1;
1871 fTTL = 30000;
1872 fSplitMaxInputFileNumber = 100;
1873 fMaxInitFailed = 0;
1874 fMasterResubmitThreshold = 0;
bb885a9e 1875 fNtestFiles = 10;
149d288c 1876 fNreplicas = 2;
a8739e8a 1877 fRunRange[0] = 0;
1878 fRunRange[1] = 0;
8e3bd06a 1879 fRunPrefix = "%d";
319593fb 1880 fNrunsPerMaster = 1;
16a4353c 1881 fMaxMergeFiles = 100;
c57f56b7 1882 fRunNumbers = "";
1883 fExecutable = "analysis.sh";
0a1c1f7f 1884 fExecutableCommand = "root -b -q";
c57f56b7 1885 fArguments = "";
631c0b05 1886 fExecutableArgs = "";
c57f56b7 1887 fAnalysisMacro = "myAnalysis.C";
1888 fAnalysisSource = "";
1889 fAdditionalLibs = "";
1890 fSplitMode = "se";
1891 fAPIVersion = "";
1892 fROOTVersion = "";
1893 fAliROOTVersion = "";
1894 fUser = ""; // Your alien user name
1895 fGridWorkingDir = "";
1896 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1897 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1898 fFriendChainName = "";
c57f56b7 1899 fGridOutputDir = "output";
b83f84f0 1900 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1901 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1902 fInputFormat = "xml-single";
1903 fJDLName = "analysis.jdl";
c6cb3634 1904 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1905 fMergeExcludes = "";
0f389141 1906 fMergeViaJDL = 0;
348be253 1907 SetUseCopy(kTRUE);
1908 SetCheckCopy(kTRUE);
149d288c 1909 SetDefaultOutputs(kTRUE);
e1c22e21 1910 fOverwriteMode = 1;
c57f56b7 1911}
1912
0f389141 1913//______________________________________________________________________________
b3e07543 1914Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1915{
b3e07543 1916// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1917 // First check if the result is already in the output directory.
1918 if (FileExists(Form("%s/%s",aliendir,filename))) {
1919 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1920 return kFALSE;
1921 }
b3e07543 1922 // Now check the last stage done.
1923 Int_t stage = 0;
1924 while (1) {
1925 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1926 stage++;
1927 }
1928 // Next stage of merging
1929 stage++;
1930 TString pattern = "*root_archive.zip";
1931 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1932 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1933 if (res) delete res;
1934 // Write standard output to file
1935 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1936 // Count the number of files inside
1937 ifstream ifile;
1938 ifile.open(Form("Stage_%d.xml",stage));
1939 if (!ifile.good()) {
1940 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1941 return kFALSE;
b3e07543 1942 }
1943 TString line;
1944 Int_t nfiles = 0;
1945 while (!ifile.eof()) {
1946 ifile >> line;
1947 if (line.Contains("/event")) nfiles++;
7c2cd90a 1948 }
b3e07543 1949 ifile.close();
1950 if (!nfiles) {
1951 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1952 return kFALSE;
1953 } else {
1954 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1955 }
b3e07543 1956 // Copy the file in the output directory
1957 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1958 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1959 // Check if this is the last stage to be done.
1960 Bool_t laststage = (nfiles<nperchunk);
1961 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1962 if (laststage) {
1963 printf("### Submiting final merging stage %d\n", stage);
1964 TString finalJDL = jdl;
1965 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1966 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1967 Int_t jobId = SubmitSingleJob(query);
1968 if (!jobId) return kFALSE;
1969 } else {
1970 printf("### Submiting merging stage %d\n", stage);
03335324 1971 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1972 Int_t jobId = SubmitSingleJob(query);
1973 if (!jobId) return kFALSE;
7c2cd90a 1974 }
b3e07543 1975 return kTRUE;
1976}
7c2cd90a 1977
1978//______________________________________________________________________________
1979Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1980{
1981// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1982 if (!gGrid) return 0;
1983 printf("=> %s ------> ",query);
1984 TGridResult *res = gGrid->Command(query);
1985 if (!res) return 0;
1986 TString jobId = res->GetKey(0,"jobId");
1987 delete res;
1988 if (jobId.IsNull()) {
1989 printf("submission failed. Reason:\n");
1990 gGrid->Stdout();
1991 gGrid->Stderr();
1992 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1993 return 0;
1994 }
1995 printf(" Job id: %s\n", jobId.Data());
1996 return atoi(jobId);
1997}
1998
1999//______________________________________________________________________________
b3e07543 2000Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2001{
b3e07543 2002// Merge given output files from basedir. Basedir can be an alien output directory
2003// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2004// files in a group (ignored for xml input). Merging can be done in stages:
2005// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2006// stage=1 : works with an xml of all root_archive.zip in the output directory
2007// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2008 TString outputFile = output;
0f389141 2009 TString command;
a2f5fc01 2010 TString outputChunk;
2011 TString previousChunk = "";
b3e07543 2012 TObjArray *listoffiles = new TObjArray();
2013// listoffiles->SetOwner();
a2f5fc01 2014 Int_t countChunk = 0;
2015 Int_t countZero = nmaxmerge;
0f389141 2016 Bool_t merged = kTRUE;
a2f5fc01 2017 Int_t index = outputFile.Index("@");
2018 if (index > 0) outputFile.Remove(index);
7c2cd90a 2019 TString inputFile = outputFile;
b3e07543 2020 TString sbasedir = basedir;
2021 if (sbasedir.Contains(".xml")) {
2022 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2023 nmaxmerge = 9999999;
2024 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2025 if (!coll) {
2026 ::Error("MergeOutput", "Input XML collection empty.");
2027 return kFALSE;
2028 }
2029 // Iterate grid collection
2030 while (coll->Next()) {
2031 TString fname = gSystem->DirName(coll->GetTURL());
2032 fname += "/";
2033 fname += inputFile;
2034 listoffiles->Add(new TNamed(fname.Data(),""));
2035 }
2036 } else {
2037 command = Form("find %s/ *%s", basedir, inputFile.Data());
2038 printf("command: %s\n", command.Data());
2039 TGridResult *res = gGrid->Command(command);
2040 if (!res) {
2041 ::Error("MergeOutput","No result for the find command\n");
2042 delete listoffiles;
2043 return kFALSE;
2044 }
2045 TIter nextmap(res);
2046 TMap *map = 0;
2047 while ((map=(TMap*)nextmap())) {
2048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2049 if (!objs || !objs->GetString().Length()) {
2050 // Nothing found - skip this output
2051 delete res;
2052 delete listoffiles;
2053 return kFALSE;
2054 }
2055 listoffiles->Add(new TNamed(objs->GetName(),""));
2056 }
2057 delete res;
2058 }
2059 if (!listoffiles->GetEntries()) {
7c2cd90a 2060 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2061 delete listoffiles;
0f389141 2062 return kFALSE;
2063 }
2064
2065 TFileMerger *fm = 0;
b3e07543 2066 TIter next0(listoffiles);
2067 TObjArray *listoffilestmp = new TObjArray();
2068 listoffilestmp->SetOwner();
2069 TObject *nextfile;
2070 TString snextfile;
2071 // Keep only the files at upper level
2072 Int_t countChar = 0;
2073 while ((nextfile=next0())) {
2074 snextfile = nextfile->GetName();
2075 Int_t crtCount = snextfile.CountChar('/');
2076 if (nextfile == listoffiles->First()) countChar = crtCount;
2077 if (crtCount < countChar) countChar = crtCount;
2078 }
2079 next0.Reset();
2080 while ((nextfile=next0())) {
2081 snextfile = nextfile->GetName();
2082 Int_t crtCount = snextfile.CountChar('/');
2083 if (crtCount > countChar) {
2084 delete nextfile;
2085 continue;
2086 }
2087 listoffilestmp->Add(nextfile);
2088 }
2089 delete listoffiles;
2090 listoffiles = listoffilestmp; // Now contains 'good' files
2091 listoffiles->Print();
2092 TIter next(listoffiles);
7c2cd90a 2093 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2094 outputChunk = outputFile;
2095 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2096 // Check for existent temporary merge files
e1c22e21 2097 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2098 // Preserve old merging functionality for stage 0.
2099 if (stage==0) {
2100 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2101 while (1) {
2102 // Skip as many input files as in a chunk
2ea55496 2103 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2104 nextfile = next();
2105 if (!nextfile) {
2ea55496 2106 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2107 delete listoffiles;
2ea55496 2108 return kFALSE;
2109 }
b3e07543 2110 snextfile = nextfile->GetName();
7c2cd90a 2111 }
2112 outputChunk = outputFile;
2113 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2114 countChunk++;
2115 if (gSystem->AccessPathName(outputChunk)) continue;
2116 // Merged file with chunks up to <countChunk> found
2117 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2118 previousChunk = outputChunk;
2119 break;
2120 }
2121 }
2122 countZero = nmaxmerge;
2123
b3e07543 2124 while ((nextfile=next())) {
2125 snextfile = nextfile->GetName();
2ea55496 2126 // Loop 'find' results and get next LFN
7c2cd90a 2127 if (countZero == nmaxmerge) {
2128 // First file in chunk - create file merger and add previous chunk if any.
2129 fm = new TFileMerger(kFALSE);
2130 fm->SetFastMethod(kTRUE);
2131 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2132 outputChunk = outputFile;
2133 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2134 }
2135 // If last file found, put merged results in the output file
b3e07543 2136 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2137 // Add file to be merged and decrement chunk counter.
b3e07543 2138 fm->AddFile(snextfile);
7c2cd90a 2139 countZero--;
b3e07543 2140 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2141 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2142 // Nothing found - skip this output
2143 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2144 merged = kFALSE;
2145 break;
7c2cd90a 2146 }
2147 fm->OutputFile(outputChunk);
2148 // Merge the outputs, then go to next chunk
2149 if (!fm->Merge()) {
2150 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2151 merged = kFALSE;
2152 break;
7c2cd90a 2153 } else {
2154 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2155 gSystem->Unlink(previousChunk);
2156 }
b3e07543 2157 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2158 countChunk++;
2159 countZero = nmaxmerge;
2160 previousChunk = outputChunk;
0f389141 2161 }
0f389141 2162 }
b3e07543 2163 delete listoffiles;
2164 delete fm;
7c2cd90a 2165 return merged;
2166 }
2167 // Merging stage different than 0.
2168 // Move to the begining of the requested chunk.
7c2cd90a 2169 fm = new TFileMerger(kFALSE);
2170 fm->SetFastMethod(kTRUE);
b3e07543 2171 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2172 delete listoffiles;
7c2cd90a 2173 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2174 // Nothing found - skip this output
2175 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2176 delete fm;
2177 return kFALSE;
2178 }
b3e07543 2179 fm->OutputFile(outputFile);
7c2cd90a 2180 // Merge the outputs
2181 if (!fm->Merge()) {
2182 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2183 delete fm;
2184 return kFALSE;
2185 } else {
b3e07543 2186 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2187 }
7c2cd90a 2188 delete fm;
2189 return kTRUE;
0f389141 2190}
2191
c57f56b7 2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::MergeOutputs()
2194{
2195// Merge analysis outputs existing in the AliEn space.
2196 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2197 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2198 if (!Connect()) {
2199 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2200 return kFALSE;
0f389141 2201 }
c9e8f7fd 2202 if (fMergeViaJDL) {
2203 if (!TestBit(AliAnalysisGrid::kMerge)) {
2204 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2205 return kFALSE;
2206 }
2207 if (fProductionMode) {
2208 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2209 return kFALSE;
2210 }
0f389141 2211 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2212 if (!SubmitMerging()) return kFALSE;
0f389141 2213 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2214 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2215 return kFALSE;
c57f56b7 2216 }
2217 // Get the output path
d2a409b2 2218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2219 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2220 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2221 return kFALSE;
2222 }
2223 if (!fOutputFiles.Length()) {
2224 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2225 return kFALSE;
0f389141 2226 }
2227 // Check if fast read option was requested
e1c22e21 2228 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2229 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2230 if (fFastReadOption) {
d3339be3 2231 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2232 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2233 gEnv->SetValue("XNet.ConnectTimeout",50);
2234 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2235 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2236 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2237 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2238 }
e8b839ab 2239 // Make sure we change the temporary directory
2240 gSystem->Setenv("TMPDIR", gSystem->pwd());
70c52cf3 2241 // Set temporary compilation directory to current one
2242 gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
149d288c 2243 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2244 TIter next(list);
2245 TObjString *str;
a2f5fc01 2246 TString outputFile;
c57f56b7 2247 Bool_t merged = kTRUE;
2248 while((str=(TObjString*)next())) {
a2f5fc01 2249 outputFile = str->GetString();
2250 Int_t index = outputFile.Index("@");
2251 if (index > 0) outputFile.Remove(index);
2252 TString outputChunk = outputFile;
2253 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2254 // Skip already merged outputs
a2f5fc01 2255 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2256 if (fOverwriteMode) {
a2f5fc01 2257 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2258 gSystem->Unlink(outputFile);
2259 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2260 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2261 outputChunk.Data());
2262 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2263 }
2264 } else {
a2f5fc01 2265 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2266 continue;
2267 }
2268 } else {
a2f5fc01 2269 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2270 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2271 outputChunk.Data());
2272 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2273 }
2274 }
bb885a9e 2275 if (fMergeExcludes.Length() &&
a2f5fc01 2276 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2277 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2278 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2279 if (!merged) {
2280 Error("MergeOutputs", "Terminate() will NOT be executed");
2281 return kFALSE;
ff07ec61 2282 }
2283 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2284 if (fileOpened) fileOpened->Close();
c57f56b7 2285 }
0f389141 2286 return kTRUE;
c57f56b7 2287}
2288
bb885a9e 2289//______________________________________________________________________________
2290void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2291{
2292// Use the output files connected to output containers from the analysis manager
2293// rather than the files defined by SetOutputFiles
2294 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2295 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2296 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2297}
2298
149d288c 2299//______________________________________________________________________________
2300void AliAnalysisAlien::SetOutputFiles(const char *list)
2301{
2302// Manually set the output files list.
2303// Removes duplicates. Not allowed if default outputs are not disabled.
2304 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2305 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2306 return;
2307 }
2308 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2309 fOutputFiles = "";
2310 TString slist = list;
2311 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2312 TObjArray *arr = slist.Tokenize(" ");
2313 TObjString *os;
2314 TIter next(arr);
2315 TString sout;
2316 while ((os=(TObjString*)next())) {
2317 sout = os->GetString();
2318 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2319 if (fOutputFiles.Contains(sout)) continue;
2320 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2321 fOutputFiles += sout;
2322 }
2323 delete arr;
f790bc1b 2324}
149d288c 2325
2326//______________________________________________________________________________
2327void AliAnalysisAlien::SetOutputArchive(const char *list)
2328{
2329// Manually set the output archive list. Free text - you are on your own...
2330// Not allowed if default outputs are not disabled.
2331 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2332 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2333 return;
2334 }
2335 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2336 fOutputArchive = list;
2337}
2338
2339//______________________________________________________________________________
2340void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2341{
2342// Setting a prefered output SE is not allowed anymore.
2343 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2344}
2345
5513444a 2346//______________________________________________________________________________
2347Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2348{
2349// Start remote grid analysis.
3bdcb562 2350 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2351 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2352 if (!mgr || !mgr->IsInitialized()) {
2353 Error("StartAnalysis", "You need an initialized analysis manager for this");
2354 return kFALSE;
2355 }
2356 // Are we in PROOF mode ?
2357 if (mgr->IsProofMode()) {
90d50a8c 2358 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2359 if (fProofCluster.IsNull()) {
2360 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2361 return kFALSE;
2362 }
2363 if (fProofDataSet.IsNull() && !testMode) {
2364 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2365 return kFALSE;
2366 }
2367 // Set the needed environment
2368 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2369 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2370 if (fProofReset && !testMode) {
2371 if (fProofReset==1) {
2372 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2373 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2374 } else {
2375 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2376 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2377 }
2378 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2379 return kFALSE;
2380 }
2381 // Do we need to change the ROOT version ? The success of this cannot be checked.
2382 if (!fRootVersionForProof.IsNull() && !testMode) {
2383 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2384 fProofCluster.Data(), fRootVersionForProof.Data()));
2385 }
2386 // Connect to PROOF and check the status
2387 Long_t proof = 0;
27734f0e 2388 TString sworkers;
2389 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2390 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2391 if (!testMode) {
27734f0e 2392 if (!sworkers.IsNull())
2393 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2394 else
2395 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2396 } else {
2397 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2398 if (!proof) {
2399 Error("StartAnalysis", "Could not start PROOF in test mode");
2400 return kFALSE;
2401 }
2402 }
2403 if (!proof) {
2404 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2405 return kFALSE;
2406 }
27734f0e 2407 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2408 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2409 // Is dataset existing ?
2410 if (!testMode) {
2411 TString dataset = fProofDataSet;
2412 Int_t index = dataset.Index("#");
2413 if (index>=0) dataset.Remove(index);
830acc4c 2414// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2415// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2416// return kFALSE;
2417// }
2418// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2419 }
2420 // Is ClearPackages() needed ?
2421 if (TestSpecialBit(kClearPackages)) {
2422 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2423 gROOT->ProcessLine("gProof->ClearPackages();");
2424 }
2425 // Is a given aliroot mode requested ?
2426 TList optionsList;
1ac846e9 2427 TString parLibs;
830acc4c 2428 if (!fAliRootMode.IsNull()) {
3bdcb562 2429 TString alirootMode = fAliRootMode;
2430 if (alirootMode == "default") alirootMode = "";
2431 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2432 optionsList.SetOwner();
2433 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2434 // Check the additional libs to be loaded
2435 TString extraLibs;
1ac846e9 2436 Bool_t parMode = kFALSE;
3bdcb562 2437 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2438 // Parse the extra libs for .so
2439 if (fAdditionalLibs.Length()) {
2440 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2441 TIter next(list);
2442 TObjString *str;
1ac846e9 2443 while((str=(TObjString*)next())) {
2444 if (str->GetString().Contains(".so")) {
2445 if (parMode) {
2446 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2447 break;
2448 }
2449 TString stmp = str->GetName();
2450 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2451 stmp.ReplaceAll(".so","");
2452 if (!extraLibs.IsNull()) extraLibs += ":";
2453 extraLibs += stmp;
2454 continue;
2455 }
2456 if (str->GetString().Contains(".par")) {
2457 // The first par file found in the list will not allow any further .so
2458 parMode = kTRUE;
2459 if (!parLibs.IsNull()) parLibs += ":";
2460 parLibs += str->GetName();
2461 continue;
2462 }
3bdcb562 2463 }
2464 if (list) delete list;
2465 }
2466 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2467 // Check extra includes
2468 if (!fIncludePath.IsNull()) {
2469 TString includePath = fIncludePath;
2470 includePath.ReplaceAll(" ",":");
8ddad121 2471 includePath.ReplaceAll("$ALICE_ROOT","");
2472 includePath.ReplaceAll("${ALICE_ROOT}","");
2473 includePath.ReplaceAll("-I","");
830acc4c 2474 includePath.Strip(TString::kTrailing, ':');
2475 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2476 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2477 }
27734f0e 2478 // Check if connection to grid is requested
2479 if (TestSpecialBit(kProofConnectGrid))
2480 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2481 // Enable AliRoot par
2482 if (testMode) {
2483 // Enable proof lite package
2484 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2485 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2486 TNamed *obj = (TNamed*)optionsList.At(i);
2487 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2488 }
2489 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2490 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2491 Info("StartAnalysis", "AliRootProofLite enabled");
2492 } else {
2493 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2494 return kFALSE;
2495 }
2496 } else {
8ddad121 2497 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2498 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2499 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2500 return kFALSE;
2501 }
27734f0e 2502 }
1ac846e9 2503 // Enable first par files from fAdditionalLibs
2504 if (!parLibs.IsNull()) {
2505 TObjArray *list = parLibs.Tokenize(":");
2506 TIter next(list);
2507 TObjString *package;
2508 while((package=(TObjString*)next())) {
40501bb7 2509 TString spkg = package->GetName();
2510 spkg.ReplaceAll(".par", "");
2511 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2512 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2513 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2514 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2515 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2516 return kFALSE;
2517 }
2518 } else {
2519 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2520 return kFALSE;
2521 }
2522 }
2523 if (list) delete list;
2524 }
3bdcb562 2525 } else {
2526 if (fAdditionalLibs.Contains(".so") && !testMode) {
2527 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2528 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2529 return kFALSE;
2530 }
2531 }
2532 // Enable par files if requested
2533 if (fPackages && fPackages->GetEntries()) {
2534 TIter next(fPackages);
2535 TObject *package;
2536 while ((package=next())) {
1ac846e9 2537 // Skip packages already enabled
2538 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2539 TString spkg = package->GetName();
2540 spkg.ReplaceAll(".par", "");
2541 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2542 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2543 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2544 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2545 return kFALSE;
2546 }
2547 } else {
2548 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2549 return kFALSE;
2550 }
2551 }
2552 }
2553 // Do we need to load analysis source files ?
2554 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2555 if (fAnalysisSource.Length()) {
2556 TObjArray *list = fAnalysisSource.Tokenize(" ");
2557 TIter next(list);
2558 TObjString *str;
2559 while((str=(TObjString*)next())) {
2560 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2561 }
2562 if (list) delete list;
2563 }
830acc4c 2564 if (testMode) {
2565 // Register dataset to proof lite.
2566 if (fFileForTestMode.IsNull()) {
2567 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2568 return kFALSE;
2569 }
2570 if (gSystem->AccessPathName(fFileForTestMode)) {
2571 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2572 return kFALSE;
2573 }
2574 TFileCollection *coll = new TFileCollection();
2575 coll->AddFromFile(fFileForTestMode);
8ddad121 2576 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2577 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2578 }
3bdcb562 2579 return kTRUE;
2580 }
c57f56b7 2581
43da816a 2582 // Check if output files have to be taken from the analysis manager
2583 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2584 // Add output files and AOD files
2585 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2586 // Add extra files registered to the analysis manager
f790bc1b 2587 TString extra = GetListOfFiles("ext");
2588 if (!extra.IsNull()) {
149d288c 2589 extra.ReplaceAll(".root", "*.root");
f790bc1b 2590 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2591 fOutputFiles += extra;
84fcd93f 2592 }
149d288c 2593 // Compose the output archive.
b83f84f0 2594 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2595 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2596 }
f7b1cbc2 2597// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2598 if (TestBit(AliAnalysisGrid::kOffline)) {
2599 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2600 \n there nor any job run. You can revise the JDL and analysis \
2601 \n macro then run the same in \"submit\" mode.");
2602 } else if (TestBit(AliAnalysisGrid::kTest)) {
2603 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2604 \n dataset.");
2605 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2606 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2607 \n space and job submitted.");
2608 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2609 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2610 if (fMergeViaJDL) CheckInputData();
5513444a 2611 return kTRUE;
c57f56b7 2612 } else {
2613 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2614 }
2615
348be253 2616 Print();
c57f56b7 2617 if (!Connect()) {
2618 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2619 return kFALSE;
16a4353c 2620 }
d3b18c4c 2621 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2622 if (!CheckInputData()) {
2623 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2624 return kFALSE;
c57f56b7 2625 }
d3339be3 2626 if (!CreateDataset(fDataPattern)) {
2627 TString serror;
2628 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2629 if (fRunNumbers.Length()) serror = "run numbers";
2630 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2631 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2632 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2633 return kFALSE;
2634 }
c57f56b7 2635 WriteAnalysisFile();
2636 WriteAnalysisMacro();
2637 WriteExecutable();
2638 WriteValidationScript();
0f389141 2639 if (fMergeViaJDL) {
2640 WriteMergingMacro();
2641 WriteMergeExecutable();
2642 WriteValidationScript(kTRUE);
2643 }
5513444a 2644 if (!CreateJDL()) return kFALSE;
2645 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2646 if (testMode) {
c57f56b7 2647 // Locally testing the analysis
2648 Info("StartAnalysis", "\n_______________________________________________________________________ \
2649 \n Running analysis script in a daughter shell as on a worker node \
2650 \n_______________________________________________________________________");
149d288c 2651 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2652 TIter next(list);
2653 TObjString *str;
a2f5fc01 2654 TString outputFile;
c57f56b7 2655 while((str=(TObjString*)next())) {
a2f5fc01 2656 outputFile = str->GetString();
2657 Int_t index = outputFile.Index("@");
2658 if (index > 0) outputFile.Remove(index);
2659 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2660 }
2661 delete list;
2662 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2663 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2664// gSystem->Exec("cat stdout");
5513444a 2665 return kFALSE;
c57f56b7 2666 }
5513444a 2667 // Check if submitting is managed by LPM manager
a3e84053 2668 if (fProductionMode) {
5513444a 2669 TString prodfile = fJDLName;
2670 prodfile.ReplaceAll(".jdl", ".prod");
2671 WriteProductionFile(prodfile);
2672 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2673 return kFALSE;
2674 }
a8739e8a 2675 // Submit AliEn job(s)
d2a409b2 2676 gGrid->Cd(fGridOutputDir);
a8739e8a 2677 TGridResult *res;
c57f56b7 2678 TString jobID = "";
d2a409b2 2679 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2680 // Submit a given xml or a set of runs
a8739e8a 2681 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2682 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2683 if (res) {
2684 const char *cjobId = res->GetKey(0,"jobId");
2685 if (!cjobId) {
a03be957 2686 gGrid->Stdout();
2687 gGrid->Stderr();
a8739e8a 2688 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2689 return kFALSE;
a8739e8a 2690 } else {
2691 Info("StartAnalysis", "\n_______________________________________________________________________ \
2692 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2693 \n_______________________________________________________________________",
2694 fJDLName.Data(), cjobId);
2695 jobID = cjobId;
2696 }
2697 delete res;
a03be957 2698 } else {
2699 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2700 return kFALSE;
a8739e8a 2701 }
2702 } else {
d2a409b2 2703 // Submit for a range of enumeration of runs.
a03be957 2704 if (!Submit()) return kFALSE;
c57f56b7 2705 }
a8739e8a 2706
c57f56b7 2707 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2708 \n You may exit at any time and terminate the job later using the option <terminate> \
2709 \n ##################################################################################", jobID.Data());
bb885a9e 2710 gSystem->Exec("aliensh");
5513444a 2711 return kTRUE;
c57f56b7 2712}
2713
f790bc1b 2714//______________________________________________________________________________
2715const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2716{
2717// Get a comma-separated list of output files of the requested type.
2718// Type can be (case unsensitive):
2719// aod - list of aod files (std, extensions and filters)
2720// out - list of output files connected to containers (but not aod's or extras)
2721// ext - list of extra files registered to the manager
2722// ter - list of files produced in terminate
2723 static TString files;
2724 files = "";
2725 TString stype = type;
2726 stype.ToLower();
2727 TString aodfiles, extra;
2728 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2729 if (!mgr) {
2730 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2731 return files.Data();
2732 }
2733 if (mgr->GetOutputEventHandler()) {
2734 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2735 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2736 if (!extraaod.IsNull()) {
2737 aodfiles += ",";
2738 aodfiles += extraaod;
2739 }
2740 }
2741 if (stype.Contains("aod")) {
2742 files = aodfiles;
2743 if (stype == "aod") return files.Data();
2744 }
2745 // Add output files that are not in the list of AOD files
2746 TString outputfiles = "";
2747 TIter next(mgr->GetOutputs());
2748 AliAnalysisDataContainer *output;
2749 const char *filename = 0;
2750 while ((output=(AliAnalysisDataContainer*)next())) {
2751 filename = output->GetFileName();
2752 if (!(strcmp(filename, "default"))) continue;
2753 if (outputfiles.Contains(filename)) continue;
2754 if (aodfiles.Contains(filename)) continue;
2755 if (!outputfiles.IsNull()) outputfiles += ",";
2756 outputfiles += filename;
2757 }
2758 if (stype.Contains("out")) {
2759 if (!files.IsNull()) files += ",";
2760 files += outputfiles;
2761 if (stype == "out") return files.Data();
2762 }
2763 // Add extra files registered to the analysis manager
2764 TString sextra;
2765 extra = mgr->GetExtraFiles();
2766 if (!extra.IsNull()) {
2767 extra.Strip();
2768 extra.ReplaceAll(" ", ",");
2769 TObjArray *fextra = extra.Tokenize(",");
2770 TIter nextx(fextra);
2771 TObject *obj;
2772 while ((obj=nextx())) {
2773 if (aodfiles.Contains(obj->GetName())) continue;
2774 if (outputfiles.Contains(obj->GetName())) continue;
2775 if (sextra.Contains(obj->GetName())) continue;
2776 if (!sextra.IsNull()) sextra += ",";
2777 sextra += obj->GetName();
2778 }
2779 delete fextra;
2780 if (stype.Contains("ext")) {
2781 if (!files.IsNull()) files += ",";
2782 files += sextra;
2783 }
2784 }
2785 if (stype == "ext") return files.Data();
2786 TString termfiles;
2787 if (!fTerminateFiles.IsNull()) {
2788 fTerminateFiles.Strip();
2789 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2790 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2791 TIter nextx(fextra);
2792 TObject *obj;
2793 while ((obj=nextx())) {
2794 if (aodfiles.Contains(obj->GetName())) continue;
2795 if (outputfiles.Contains(obj->GetName())) continue;
2796 if (termfiles.Contains(obj->GetName())) continue;
2797 if (sextra.Contains(obj->GetName())) continue;
2798 if (!termfiles.IsNull()) termfiles += ",";
2799 termfiles += obj->GetName();
2800 }
2801 delete fextra;
2802 }
2803 if (stype.Contains("ter")) {
2804 if (!files.IsNull() && !termfiles.IsNull()) {
2805 files += ",";
2806 files += termfiles;
2807 }
2808 }
2809 return files.Data();
2810}
2811
d2a409b2 2812//______________________________________________________________________________
a03be957 2813Bool_t AliAnalysisAlien::Submit()
d2a409b2 2814{
2815// Submit all master jobs.
2816 Int_t nmasterjobs = fInputFiles->GetEntries();
2817 Long_t tshoot = gSystem->Now();
a03be957 2818 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2819 while (fNsubmitted < nmasterjobs) {
2820 Long_t now = gSystem->Now();
2821 if ((now-tshoot)>30000) {
2822 tshoot = now;
a03be957 2823 if (!SubmitNext()) return kFALSE;
d2a409b2 2824 }
2825 }
a03be957 2826 return kTRUE;
d2a409b2 2827}
2828
0f389141 2829//______________________________________________________________________________
a03be957 2830Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2831{
2832// Submit all merging jobs.
2833 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2834 gGrid->Cd(fGridOutputDir);
2835 TString mergeJDLName = fExecutable;
2836 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2837 if (!fInputFiles) {
2838 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2839 return kFALSE;
2840 }
0f389141 2841 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2842 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2843 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2844 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2845 if (fOutputToRunNo) {
2846 // The output directory is the run number
2847 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2848 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2849 } else {
17b4e9a4 2850 if (!fRunNumbers.Length() && !fRunRange[0]) {
2851 // The output directory is the grid outdir
2852 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2853 runOutDir = fGridOutputDir;
2854 } else {
2855 // The output directory is the master number in 3 digits format
2856 printf("### Submitting merging job for master <%03d>\n", i);
2857 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2858 }
7c2cd90a 2859 }
2860 // Check now the number of merging stages.
2b222f7c 2861 TObjArray *list = fOutputFiles.Tokenize(",");
2862 TIter next(list);
2863 TObjString *str;
2864 TString outputFile;
2865 while((str=(TObjString*)next())) {
2866 outputFile = str->GetString();
2867 Int_t index = outputFile.Index("@");
2868 if (index > 0) outputFile.Remove(index);
2869 if (!fMergeExcludes.Contains(outputFile)) break;
2870 }
2871 delete list;
b3e07543 2872 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2873 if (!done && (i==ntosubmit-1)) return kFALSE;
17b4e9a4 2874 if (!fRunNumbers.Length() && !fRunRange[0]) break;
0f389141 2875 }
a03be957 2876 if (!ntosubmit) return kTRUE;
9f020952 2877 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2878 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2879 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2880 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2881 \n ################################################################################################################");
0f389141 2882 gSystem->Exec("aliensh");
a03be957 2883 return kTRUE;
0f389141 2884}
2885
d2a409b2 2886//______________________________________________________________________________
a03be957 2887Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2888{
08d5b699 2889// Submit next bunch of master jobs if the queue is free. The first master job is
2890// submitted right away, while the next will not be unless the previous was split.
2891// The plugin will not submit new master jobs if there are more that 500 jobs in
2892// waiting phase.
d2a409b2 2893 static Bool_t iscalled = kFALSE;
2894 static Int_t firstmaster = 0;
2895 static Int_t lastmaster = 0;
2896 static Int_t npermaster = 0;
a03be957 2897 if (iscalled) return kTRUE;
d2a409b2 2898 iscalled = kTRUE;
2899 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2900 Int_t ntosubmit = 0;
2901 TGridResult *res;
2902 TString jobID = "";
3bdcb562 2903 Int_t nmasterjobs = fInputFiles->GetEntries();
2904 if (!fNsubmitted) {
2905 ntosubmit = 1;
2906 if (!IsUseSubmitPolicy()) {
90d50a8c 2907 if (nmasterjobs>5)
3bdcb562 2908 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2909 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2910 ntosubmit = nmasterjobs;
2911 }
2912 } else {
d2a409b2 2913 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2914 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2915 // If last master not split, just return
a03be957 2916 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2917 // No more than 100 waiting jobs
08d5b699 2918 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2919 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2920 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2921 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2922 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2923 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2924 }
d2a409b2 2925 for (Int_t i=0; i<ntosubmit; i++) {
2926 // Submit for a range of enumeration of runs.
a03be957 2927 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2928 TString query;
cd11251e 2929 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2930 runOutDir.ReplaceAll(".xml", "");
2931 if (fOutputToRunNo)
2932 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2933 else
2934 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2935 printf("********* %s\n",query.Data());
d2a409b2 2936 res = gGrid->Command(query);
2937 if (res) {
98ca124f 2938 TString cjobId1 = res->GetKey(0,"jobId");
2939 if (!cjobId1.Length()) {
d2a409b2 2940 iscalled = kFALSE;
a03be957 2941 gGrid->Stdout();
2942 gGrid->Stderr();
2943 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2944 return kFALSE;
d2a409b2 2945 } else {
2946 Info("StartAnalysis", "\n_______________________________________________________________________ \
2947 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2948 \n_______________________________________________________________________",
98ca124f 2949 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2950 jobID += cjobId1;
2951 jobID += " ";
98ca124f 2952 lastmaster = cjobId1.Atoi();
d2a409b2 2953 if (!firstmaster) firstmaster = lastmaster;
2954 fNsubmitted++;
2955 }
2956 delete res;
a03be957 2957 } else {
2958 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2959 return kFALSE;
d2a409b2 2960 }
2961 }
2962 iscalled = kFALSE;
a03be957 2963 return kTRUE;
d2a409b2 2964}
2965
c57f56b7 2966//______________________________________________________________________________
2967void AliAnalysisAlien::WriteAnalysisFile()
2968{
f10e8481 2969// Write current analysis manager into the file <analysisFile>
2970 TString analysisFile = fExecutable;
2971 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2972 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2973 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2974 if (!mgr || !mgr->IsInitialized()) {
2975 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2976 return;
2977 }
2978 // Check analysis type
2979 TObject *handler;
2980 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2981 handler = (TObject*)mgr->GetInputEventHandler();
2982 if (handler) {
2983 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2984 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2985 }
2986 TDirectory *cdir = gDirectory;
f10e8481 2987 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2988 if (file) {
e16a394c 2989 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2990 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2991 // Unless merging makes no sense
2992 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2993 mgr->Write();
2994 delete file;
fe2d7fc2 2995 // Enable termination for local jobs
2996 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2997 }
2998 if (cdir) cdir->cd();
f10e8481 2999 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 3000 }
3001 Bool_t copy = kTRUE;
d3b18c4c 3002 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3003 if (copy) {
3004 CdWork();
3005 TString workdir = gGrid->GetHomeDirectory();
3006 workdir += fGridWorkingDir;
d3b18c4c 3007 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 3008 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3009 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3010 }
3011}
3012
3013//______________________________________________________________________________
3014void AliAnalysisAlien::WriteAnalysisMacro()
3015{
3016// Write the analysis macro that will steer the analysis in grid mode.
3017 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3018 ofstream out;
3019 out.open(fAnalysisMacro.Data(), ios::out);
3020 if (!out.good()) {
3021 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3022 return;
3023 }
5b273635 3024 Bool_t hasSTEERBase = kFALSE;
3025 Bool_t hasESD = kFALSE;
3026 Bool_t hasAOD = kFALSE;
3027 Bool_t hasANALYSIS = kFALSE;
3028 Bool_t hasANALYSISalice = kFALSE;
3029 Bool_t hasCORRFW = kFALSE;
c57f56b7 3030 TString func = fAnalysisMacro;
3031 TString type = "ESD";
3032 TString comment = "// Analysis using ";
c85cfc0f 3033 if (IsUseMCchain()) {
3034 type = "MC";
3035 comment += "MC";
3036 } else {
3037 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3038 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3039 type = "AOD";
3040 comment += "AOD";
3041 }
3042 }
0df6ccf2 3043 if (type!="AOD" && fFriendChainName!="") {
3044 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3045 return;
3046 }
c57f56b7 3047 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3048 else comment += " data";
3049 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3050 func.ReplaceAll(".C", "");
3051 out << "void " << func.Data() << "()" << endl;
3052 out << "{" << endl;
3053 out << comment.Data() << endl;
3054 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3055 out << " TStopwatch timer;" << endl;
3056 out << " timer.Start();" << endl << endl;
13f0c00b 3057 // Change temp directory to current one
3058 out << "// Set temporary merging directory to current one" << endl;
3059 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3060 out << "// Set temporary compilation directory to current one" << endl;
3061 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
b3e07543 3062 // Reset existing include path
3063 out << "// Reset existing include path and add current directory first in the search" << endl;
3064 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3065 if (!fExecutableCommand.Contains("aliroot")) {
3066 out << "// load base root libraries" << endl;
3067 out << " gSystem->Load(\"libTree\");" << endl;
3068 out << " gSystem->Load(\"libGeom\");" << endl;
3069 out << " gSystem->Load(\"libVMC\");" << endl;
3070 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3071 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3072 }
d5c6455a 3073 if (fAdditionalRootLibs.Length()) {
47a4137d 3074 // in principle libtree /lib geom libvmc etc. can go into this list, too
3075 out << "// Add aditional libraries" << endl;
3076 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3077 TIter next(list);
3078 TObjString *str;
3079 while((str=(TObjString*)next())) {
3080 if (str->GetString().Contains(".so"))
3081 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3082 }
47a4137d 3083 if (list) delete list;
d5c6455a 3084 }
57377eb5 3085 out << "// Load analysis framework libraries" << endl;
652561d5 3086 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3087 if (!fPackages) {
f790bc1b 3088 if (!fExecutableCommand.Contains("aliroot")) {
3089 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3090 out << " gSystem->Load(\"libESD\");" << endl;
3091 out << " gSystem->Load(\"libAOD\");" << endl;
3092 }
4e5c5506 3093 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3094 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3095 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3096 } else {
4e5c5506 3097 TIter next(fPackages);
3098 TObject *obj;
57377eb5 3099 TString pkgname;
57377eb5 3100 while ((obj=next())) {
3101 pkgname = obj->GetName();
4478e6f1 3102 if (pkgname == "STEERBase" ||
3103 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3104 if (pkgname == "ESD" ||
3105 pkgname == "ESD.par") hasESD = kTRUE;
3106 if (pkgname == "AOD" ||
3107 pkgname == "AOD.par") hasAOD = kTRUE;
3108 if (pkgname == "ANALYSIS" ||
3109 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3110 if (pkgname == "ANALYSISalice" ||
3111 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3112 if (pkgname == "CORRFW" ||
3113 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3114 }
3115 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3116 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3117 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3118 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3119 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3120 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3121 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3122 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3123 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3124 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3125 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3126 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3127 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3128 out << "// Compile other par packages" << endl;
3129 next.Reset();
fcc9bb6f 3130 while ((obj=next())) {
3131 pkgname = obj->GetName();
4478e6f1 3132 if (pkgname == "STEERBase" ||
3133 pkgname == "STEERBase.par" ||
3134 pkgname == "ESD" ||
3135 pkgname == "ESD.par" ||
3136 pkgname == "AOD" ||
3137 pkgname == "AOD.par" ||
3138 pkgname == "ANALYSIS" ||
3139 pkgname == "ANALYSIS.par" ||
3140 pkgname == "ANALYSISalice" ||
3141 pkgname == "ANALYSISalice.par" ||
3142 pkgname == "CORRFW" ||
3143 pkgname == "CORRFW.par") continue;
5b273635 3144 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3145 }
4e5c5506 3146 }
b3e07543 3147 out << "// include path" << endl;
3148 // Get the include path from the interpreter and remove entries pointing to AliRoot
3149 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3150 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3151 out << " TIter nextpath(listpaths);" << endl;
3152 out << " TObjString *pname;" << endl;
3153 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3154 out << " TString current = pname->GetName();" << endl;
3155 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3156 out << " gSystem->AddIncludePath(current);" << endl;
3157 out << " }" << endl;
3158 out << " if (listpaths) delete listpaths;" << endl;
3159 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3160 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3161 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3162 if (fAdditionalLibs.Length()) {
3163 out << "// Add aditional AliRoot libraries" << endl;
3164 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3165 TIter next(list);
3166 TObjString *str;
3167 while((str=(TObjString*)next())) {
3168 if (str->GetString().Contains(".so"))
3169 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3170 if (str->GetString().Contains(".par"))
3171 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3172 }
3173 if (list) delete list;
3174 }
3175 out << endl;
c57f56b7 3176 out << "// analysis source to be compiled at runtime (if any)" << endl;
3177 if (fAnalysisSource.Length()) {
3178 TObjArray *list = fAnalysisSource.Tokenize(" ");
3179 TIter next(list);
3180 TObjString *str;
3181 while((str=(TObjString*)next())) {
3182 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3183 }
3184 if (list) delete list;
3185 }
3186 out << endl;
b3e07543 3187// out << " printf(\"Currently load libraries:\\n\");" << endl;
3188// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3189 if (fFastReadOption) {
d3339be3 3190 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3191 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3192 out << "// fast xrootd reading enabled" << endl;
3193 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3194 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3195 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3196 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3197 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3198 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3199 }
c57f56b7 3200 out << "// connect to AliEn and make the chain" << endl;
3201 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3202 out << "// read the analysis manager from file" << endl;
f10e8481 3203 TString analysisFile = fExecutable;
3204 analysisFile.ReplaceAll(".sh", ".root");
3205 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3206 out << " if (!file) return;" << endl;
3207 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3208 out << " AliAnalysisManager *mgr = 0;" << endl;
3209 out << " TKey *key;" << endl;
3210 out << " while ((key=(TKey*)nextkey())) {" << endl;
3211 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3212 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3213 out << " };" << endl;
3214 out << " if (!mgr) {" << endl;
205b201f 3215 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3216 out << " return;" << endl;
3217 out << " }" << endl << endl;
3218 out << " mgr->PrintStatus();" << endl;
52b6a92b 3219 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3220 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3221 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3222 } else {
9a8288ff 3223 if (TestBit(AliAnalysisGrid::kTest))
3224 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3225 else
3226 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3227 }
3228 }
242accb2 3229 if (IsUsingTags()) {
3230 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3231 } else {
3232 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3233 }
c57f56b7 3234 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3235 out << " timer.Stop();" << endl;
3236 out << " timer.Print();" << endl;
c57f56b7 3237 out << "}" << endl << endl;
3238 if (IsUsingTags()) {
3239 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3240 out << "{" << endl;
3241 out << "// Create a chain using tags from the xml file." << endl;
3242 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3243 out << " if (!coll) {" << endl;
3244 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3245 out << " return NULL;" << endl;
3246 out << " }" << endl;
3247 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3248 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3249 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3250 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3251 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3252 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3253 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3254 out << " // Check if the cuts configuration file was provided" << endl;
3255 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3256 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3257 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3258 out << " }" << endl;
0df6ccf2 3259 if (fFriendChainName=="") {
3260 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3261 } else {
3262 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3263 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3264 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3265 }
c57f56b7 3266 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3267 out << " chain->ls();" << endl;
3268 out << " return chain;" << endl;
fcc9bb6f 3269 out << "}" << endl << endl;
c57f56b7 3270 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3271 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3272 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3273 msg += " AliLHCTagCuts *lhcCuts,\n";
3274 msg += " AliDetectorTagCuts *detCuts,\n";
3275 msg += " AliEventTagCuts *evCuts)";
23329835 3276 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3277 }
0df6ccf2 3278 }
3279 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3280 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3281 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3282 out << "{" << endl;
3283 out << "// Create a chain using url's from xml file" << endl;
242accb2 3284 out << " TString filename;" << endl;
3285 out << " Int_t run = 0;" << endl;
c85cfc0f 3286 if (IsUseMCchain()) {
3287 out << " TString treename = \"TE\";" << endl;
3288 } else {
3289 out << " TString treename = type;" << endl;
3290 out << " treename.ToLower();" << endl;
3291 out << " treename += \"Tree\";" << endl;
3292 }
e02fee64 3293 out << " printf(\"***************************************\\n\");" << endl;
3294 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3295 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3296 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3297 out << " if (!coll) {" << endl;
3298 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3299 out << " return NULL;" << endl;
3300 out << " }" << endl;
242accb2 3301 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3302 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3303 if(fFriendChainName!="") {
3304 out << " TChain *chainFriend = new TChain(treename);" << endl;
3305 }
c57f56b7 3306 out << " coll->Reset();" << endl;
0df6ccf2 3307 out << " while (coll->Next()) {" << endl;
242accb2 3308 out << " filename = coll->GetTURL("");" << endl;
3309 out << " if (mgr) {" << endl;
3310 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3311 out << " if (nrun && nrun != run) {" << endl;
3312 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3313 out << " mgr->SetRunFromPath(nrun);" << endl;
3314 out << " run = nrun;" << endl;
3315 out << " }" << endl;
3316 out << " }" << endl;
3317 out << " chain->Add(filename);" << endl;
0df6ccf2 3318 if(fFriendChainName!="") {
3319 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3320 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3321 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3322 out << " chainFriend->Add(fileFriend.Data());" << endl;
3323 }
3324 out << " }" << endl;
c57f56b7 3325 out << " if (!chain->GetNtrees()) {" << endl;
3326 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3327 out << " return NULL;" << endl;
3328 out << " }" << endl;
0df6ccf2 3329 if(fFriendChainName!="") {
3330 out << " chain->AddFriend(chainFriend);" << endl;
3331 }
c57f56b7 3332 out << " return chain;" << endl;
fcc9bb6f 3333 out << "}" << endl << endl;
c57f56b7 3334 }
5b273635 3335 if (hasANALYSISalice) {
3336 out <<"//________________________________________________________________________________" << endl;
3337 out << "Bool_t SetupPar(const char *package) {" << endl;
3338 out << "// Compile the package and set it up." << endl;
3339 out << " TString pkgdir = package;" << endl;
3340 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3341 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3342 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3343 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3344 out << " // Check for BUILD.sh and execute" << endl;
3345 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3346 out << " printf(\"*******************************\\n\");" << endl;
3347 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3348 out << " printf(\"*******************************\\n\");" << endl;
3349 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3350 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3351 out << " gSystem->ChangeDirectory(cdir);" << endl;
3352 out << " return kFALSE;" << endl;
3353 out << " }" << endl;
3354 out << " } else {" << endl;
3355 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3356 out << " gSystem->ChangeDirectory(cdir);" << endl;
3357 out << " return kFALSE;" << endl;
3358 out << " }" << endl;
3359 out << " // Check for SETUP.C and execute" << endl;
3360 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3361 out << " printf(\"*******************************\\n\");" << endl;
3362 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3363 out << " printf(\"*******************************\\n\");" << endl;
3364 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3365 out << " } else {" << endl;
3366 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3367 out << " gSystem->ChangeDirectory(cdir);" << endl;
3368 out << " return kFALSE;" << endl;
3369 out << " }" << endl;
3370 out << " // Restore original workdir" << endl;
3371 out << " gSystem->ChangeDirectory(cdir);" << endl;
3372 out << " return kTRUE;" << endl;
3373 out << "}" << endl;
3374 }
c57f56b7 3375 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3376 }
3377 Bool_t copy = kTRUE;
d3b18c4c 3378 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3379 if (copy) {
3380 CdWork();
3381 TString workdir = gGrid->GetHomeDirectory();
3382 workdir += fGridWorkingDir;
3383 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3384 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3385 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3386 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3387 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3388 }
3389 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3390 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3391 }
3392}
3393
0f389141 3394//______________________________________________________________________________
3395void AliAnalysisAlien::WriteMergingMacro()
3396{
3397// Write a macro to merge the outputs per master job.
3398 if (!fMergeViaJDL) return;
3399 if (!fOutputFiles.Length()) {
3400 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3401 return;
3402 }
3403 TString mergingMacro = fExecutable;
3404 mergingMacro.ReplaceAll(".sh","_merge.C");
3405 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3406 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3407 ofstream out;
3408 out.open(mergingMacro.Data(), ios::out);
3409 if (!out.good()) {
3410 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3411 return;
3412 }
5b273635 3413 Bool_t hasSTEERBase = kFALSE;
3414 Bool_t hasESD = kFALSE;
3415 Bool_t hasAOD = kFALSE;
3416 Bool_t hasANALYSIS = kFALSE;
3417 Bool_t hasANALYSISalice = kFALSE;
3418 Bool_t hasCORRFW = kFALSE;
0f389141 3419 TString func = mergingMacro;
3420 TString comment;
3421 func.ReplaceAll(".C", "");
f47d5cb4 3422 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3423 out << "{" << endl;
3424 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3425 out << " TStopwatch timer;" << endl;
3426 out << " timer.Start();" << endl << endl;
b3e07543 3427 // Reset existing include path
3428 out << "// Reset existing include path and add current directory first in the search" << endl;
3429 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3430 if (!fExecutableCommand.Contains("aliroot")) {
3431 out << "// load base root libraries" << endl;
3432 out << " gSystem->Load(\"libTree\");" << endl;
3433 out << " gSystem->Load(\"libGeom\");" << endl;
3434 out << " gSystem->Load(\"libVMC\");" << endl;
3435 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3436 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3437 }
0f389141 3438 if (fAdditionalRootLibs.Length()) {
3439 // in principle libtree /lib geom libvmc etc. can go into this list, too
3440 out << "// Add aditional libraries" << endl;
3441 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3442 TIter next(list);
3443 TObjString *str;
3444 while((str=(TObjString*)next())) {
3445 if (str->GetString().Contains(".so"))
3446 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3447 }
3448 if (list) delete list;
3449 }
0f389141 3450 out << "// Load analysis framework libraries" << endl;
3451 if (!fPackages) {
7c2cd90a 3452 if (!fExecutableCommand.Contains("aliroot")) {
3453 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3454 out << " gSystem->Load(\"libESD\");" << endl;
3455 out << " gSystem->Load(\"libAOD\");" << endl;
3456 }
0f389141 3457 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3458 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3459 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3460 } else {
3461 TIter next(fPackages);
3462 TObject *obj;
3463 TString pkgname;
5b273635 3464 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3465 while ((obj=next())) {
3466 pkgname = obj->GetName();
3467 if (pkgname == "STEERBase" ||
3468 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3469 if (pkgname == "ESD" ||
3470 pkgname == "ESD.par") hasESD = kTRUE;
3471 if (pkgname == "AOD" ||
3472 pkgname == "AOD.par") hasAOD = kTRUE;
3473 if (pkgname == "ANALYSIS" ||
3474 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3475 if (pkgname == "ANALYSISalice" ||
3476 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3477 if (pkgname == "CORRFW" ||
3478 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3479 }
5b273635 3480 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3481 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3482 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3483 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3484 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3485 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3486 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3487 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3488 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3489 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3490 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3491 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3492 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3493 out << "// Compile other par packages" << endl;
3494 next.Reset();
3495 while ((obj=next())) {
3496 pkgname = obj->GetName();
3497 if (pkgname == "STEERBase" ||
3498 pkgname == "STEERBase.par" ||
3499 pkgname == "ESD" ||
3500 pkgname == "ESD.par" ||
3501 pkgname == "AOD" ||
3502 pkgname == "AOD.par" ||
3503 pkgname == "ANALYSIS" ||
3504 pkgname == "ANALYSIS.par" ||
3505 pkgname == "ANALYSISalice" ||
3506 pkgname == "ANALYSISalice.par" ||
3507 pkgname == "CORRFW" ||
3508 pkgname == "CORRFW.par") continue;
5b273635 3509 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3510 }
3511 }
b3e07543 3512 out << "// include path" << endl;
3513 // Get the include path from the interpreter and remove entries pointing to AliRoot
3514 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3515 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3516 out << " TIter nextpath(listpaths);" << endl;
3517 out << " TObjString *pname;" << endl;
3518 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3519 out << " TString current = pname->GetName();" << endl;
3520 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3521 out << " gSystem->AddIncludePath(current);" << endl;
3522 out << " }" << endl;
3523 out << " if (listpaths) delete listpaths;" << endl;
3524 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3525 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3526 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3527 if (fAdditionalLibs.Length()) {
3528 out << "// Add aditional AliRoot libraries" << endl;
3529 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3530 TIter next(list);
3531 TObjString *str;
3532 while((str=(TObjString*)next())) {
3533 if (str->GetString().Contains(".so"))
3534 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3535 }
3536 if (list) delete list;
3537 }
3538 out << endl;
3539 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3540 if (fAnalysisSource.Length()) {
3541 TObjArray *list = fAnalysisSource.Tokenize(" ");
3542 TIter next(list);
3543 TObjString *str;
3544 while((str=(TObjString*)next())) {
3545 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3546 }
3547 if (list) delete list;
3548 }
149d288c 3549 out << endl;
3550
0f389141 3551 if (fFastReadOption) {
3552 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3553 out << "// fast xrootd reading enabled" << endl;
3554 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3555 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3556 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3557 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3558 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3559 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3560 }
3561 // Change temp directory to current one
3562 out << "// Set temporary merging directory to current one" << endl;
3563 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
70c52cf3 3564 out << "// Set temporary compilation directory to current one" << endl;
3565 out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
0f389141 3566 out << "// Connect to AliEn" << endl;
3567 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3568 out << " TString outputDir = dir;" << endl;
f790bc1b 3569 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3570 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3571 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3572 out << " TIter *iter = new TIter(list);" << endl;
3573 out << " TObjString *str;" << endl;
a2f5fc01 3574 out << " TString outputFile;" << endl;
0f389141 3575 out << " Bool_t merged = kTRUE;" << endl;
3576 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3577 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3578 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3579 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3580 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3581 out << " // Skip already merged outputs" << endl;
a2f5fc01 3582 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3583 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3584 out << " continue;" << endl;
3585 out << " }" << endl;
a2f5fc01 3586 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3587 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3588 out << " if (!merged) {" << endl;
a2f5fc01 3589 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3590 out << " return;" << endl;
0f389141 3591 out << " }" << endl;
3592 out << " }" << endl;
7c2cd90a 3593 out << " // all outputs merged, validate" << endl;
3594 out << " ofstream out;" << endl;
3595 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3596 out << " out.close();" << endl;
3597 out << " // read the analysis manager from file" << endl;
f866cba5 3598 TString analysisFile = fExecutable;
3599 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3600 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3601 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3602 out << " if (!file) return;" << endl;
f866cba5 3603 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3604 out << " AliAnalysisManager *mgr = 0;" << endl;
3605 out << " TKey *key;" << endl;
3606 out << " while ((key=(TKey*)nextkey())) {" << endl;
3607 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3608 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3609 out << " };" << endl;
3610 out << " if (!mgr) {" << endl;
3611 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3612 out << " return;" << endl;
3613 out << " }" << endl << endl;
5c4250fc 3614 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3615 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3616 out << " mgr->PrintStatus();" << endl;
3617 if (AliAnalysisManager::GetAnalysisManager()) {
3618 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3619 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3620 } else {
9a8288ff 3621 if (TestBit(AliAnalysisGrid::kTest))
3622 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3623 else
3624 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3625 }
3626 }
9a8288ff 3627 out << " TTree *tree = NULL;" << endl;
3628 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3629 out << "}" << endl << endl;
5b273635 3630 if (hasANALYSISalice) {
3631 out <<"//________________________________________________________________________________" << endl;
3632 out << "Bool_t SetupPar(const char *package) {" << endl;
3633 out << "// Compile the package and set it up." << endl;
3634 out << " TString pkgdir = package;" << endl;
3635 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3636 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3637 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3638 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3639 out << " // Check for BUILD.sh and execute" << endl;
3640 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3641 out << " printf(\"*******************************\\n\");" << endl;
3642 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3643 out << " printf(\"*******************************\\n\");" << endl;
3644 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3645 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3646 out << " gSystem->ChangeDirectory(cdir);" << endl;
3647 out << " return kFALSE;" << endl;
3648 out << " }" << endl;
3649 out << " } else {" << endl;
3650 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3651 out << " gSystem->ChangeDirectory(cdir);" << endl;
3652 out << " return kFALSE;" << endl;
3653 out << " }" << endl;
3654 out << " // Check for SETUP.C and execute" << endl;
3655 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3656 out << " printf(\"*******************************\\n\");" << endl;
3657 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3658 out << " printf(\"*******************************\\n\");" << endl;
3659 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3660 out << " } else {" << endl;
3661 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3662 out << " gSystem->ChangeDirectory(cdir);" << endl;
3663 out << " return kFALSE;" << endl;
3664 out << " }" << endl;
3665 out << " // Restore original workdir" << endl;
3666 out << " gSystem->ChangeDirectory(cdir);" << endl;
3667 out << " return kTRUE;" << endl;
3668 out << "}" << endl;
3669 }
0f389141 3670 }
3671 Bool_t copy = kTRUE;
d3b18c4c 3672 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3673 if (copy) {
3674 CdWork();
3675 TString workdir = gGrid->GetHomeDirectory();
3676 workdir += fGridWorkingDir;
3677 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3678 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3679 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3680 }
3681}
3682
3683//______________________________________________________________________________
3684Bool_t AliAnalysisAlien::SetupPar(const char *package)
3685{
205b201f 3686// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3687// Note that for loading the compiled library. The current directory should have precedence in
3688// LD_LIBRARY_PATH
3689 TString pkgdir = package;
3690 pkgdir.ReplaceAll(".par","");
b50a41c3 3691 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3692 TString cdir = gSystem->WorkingDirectory();
3693 gSystem->ChangeDirectory(pkgdir);
3694 // Check for BUILD.sh and execute
3695 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3696 printf("**************************************************\n");
3697 printf("*** Building PAR archive %s\n", package);
3698 printf("**************************************************\n");
3699 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3700 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3701 gSystem->ChangeDirectory(cdir);
3702 return kFALSE;
3703 }
3704 } else {
3705 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3706 gSystem->ChangeDirectory(cdir);
3707 return kFALSE;
3708 }
3709 // Check for SETUP.C and execute
3710 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3711 printf("**************************************************\n");
3712 printf("*** Setup PAR archive %s\n", package);
3713 printf("**************************************************\n");
3714 gROOT->Macro("PROOF-INF/SETUP.C");
3715 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3716 } else {
3717 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3718 gSystem->ChangeDirectory(cdir);
3719 return kFALSE;
3720 }
3721 // Restore original workdir
3722 gSystem->ChangeDirectory(cdir);
3723 return kTRUE;
3724}
3725
c57f56b7 3726//______________________________________________________________________________
3727void AliAnalysisAlien::WriteExecutable()
3728{
3729// Generate the alien executable script.
3730 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3731 ofstream out;
3732 out.open(fExecutable.Data(), ios::out);
3733 if (out.bad()) {
5513444a 3734 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3735 return;
3736 }
3737 out << "#!/bin/bash" << endl;
2a4d5166 3738 // Make sure we can properly compile par files
3739 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3740 out << "echo \"=========================================\"" << endl;
3741 out << "echo \"############## PATH : ##############\"" << endl;
3742 out << "echo $PATH" << endl;
3743 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3744 out << "echo $LD_LIBRARY_PATH" << endl;
3745 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3746 out << "echo $ROOTSYS" << endl;
3747 out << "echo \"############## which root : ##############\"" << endl;
3748 out << "which root" << endl;
3749 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3750 out << "echo $ALICE_ROOT" << endl;
3751 out << "echo \"############## which aliroot : ##############\"" << endl;
3752 out << "which aliroot" << endl;
9c5ddadc 3753 out << "echo \"############## system limits : ##############\"" << endl;
3754 out << "ulimit -a" << endl;
3755 out << "echo \"############## memory : ##############\"" << endl;
3756 out << "free -m" << endl;
c57f56b7 3757 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3758 out << fExecutableCommand << " ";
631c0b05 3759 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3760 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3761 out << "echo \"############## memory after: ##############\"" << endl;
3762 out << "free -m" << endl;
c57f56b7 3763 }
3764 Bool_t copy = kTRUE;
d3b18c4c 3765 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3766 if (copy) {
3767 CdWork();
3768 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3769 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3770 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3771 workdir += fGridWorkingDir;
3772 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3773 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3774 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3775 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3776 }
3777}
3778
0f389141 3779//______________________________________________________________________________
3780void AliAnalysisAlien::WriteMergeExecutable()
3781{
3782// Generate the alien executable script for the merging job.
3783 if (!fMergeViaJDL) return;
3784 TString mergeExec = fExecutable;
3785 mergeExec.ReplaceAll(".sh", "_merge.sh");
3786 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3787 ofstream out;
3788 out.open(mergeExec.Data(), ios::out);
3789 if (out.bad()) {
3790 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3791 return;
3792 }
3793 out << "#!/bin/bash" << endl;
2a4d5166 3794 // Make sure we can properly compile par files
3795 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3796 out << "echo \"=========================================\"" << endl;
3797 out << "echo \"############## PATH : ##############\"" << endl;
3798 out << "echo $PATH" << endl;
3799 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3800 out << "echo $LD_LIBRARY_PATH" << endl;
3801 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3802 out << "echo $ROOTSYS" << endl;
3803 out << "echo \"############## which root : ##############\"" << endl;
3804 out << "which root" << endl;
3805 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3806 out << "echo $ALICE_ROOT" << endl;
3807 out << "echo \"############## which aliroot : ##############\"" << endl;
3808 out << "which aliroot" << endl;
3809 out << "echo \"############## system limits : ##############\"" << endl;
3810 out << "ulimit -a" << endl;
3811 out << "echo \"############## memory : ##############\"" << endl;
3812 out << "free -m" << endl;
3813 out << "echo \"=========================================\"" << endl << endl;
0f389141 3814 TString mergeMacro = fExecutable;
3815 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3816 if (IsOneStageMerging())
3817 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3818 else
f47d5cb4 3819 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3820 out << fExecutableCommand << " " << "$ARG" << endl;
3821 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3822 out << "echo \"############## memory after: ##############\"" << endl;
3823 out << "free -m" << endl;
0f389141 3824 }
3825 Bool_t copy = kTRUE;
d3b18c4c 3826 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3827 if (copy) {
3828 CdWork();
3829 TString workdir = gGrid->GetHomeDirectory();
3830 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3831 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3832 workdir += fGridWorkingDir;
3833 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3834 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3835 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3836 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3837 }
3838}
3839
c57f56b7 3840//______________________________________________________________________________
5513444a 3841void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3842{
3843// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3844// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3845// Next lines: full_path_to_dataset XXX (XXX is a string)
3846// To submit, one has to: submit jdl XXX for all lines
3847 ofstream out;
3848 out.open(filename, ios::out);
3849 if (out.bad()) {
3850 Error("WriteProductionFile", "Bad file name: %s", filename);
3851 return;
3852 }
d3b18c4c 3853 TString workdir;
3854 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3855 workdir = gGrid->GetHomeDirectory();
5513444a 3856 workdir += fGridWorkingDir;
f5e8c702 3857 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3858 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3859 out << locjdl << " " << njobspermaster << endl;
5513444a 3860 Int_t nmasterjobs = fInputFiles->GetEntries();
3861 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3862 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3863 runOutDir.ReplaceAll(".xml", "");
3864 if (fOutputToRunNo)
3865 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3866 else
3867 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3868 }
d3b18c4c 3869 if (gGrid) {
3870 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3871 if (FileExists(filename)) gGrid->Rm(filename);
3872 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3873 }
5513444a 3874}
3875
3876//______________________________________________________________________________
0f389141 3877void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3878{
3879// Generate the alien validation script.
3880 // Generate the validation script
3881 TObjString *os;
d3b18c4c 3882 if (fValidationScript.IsNull()) {
3883 fValidationScript = fExecutable;
3884 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3885 }
3886 TString validationScript = fValidationScript;
3887 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3888 if (!Connect()) {
3889 Error("WriteValidationScript", "Alien connection required");
3890 return;
3891 }
d3b18c4c 3892 if (!fTerminateFiles.IsNull()) {
3893 fTerminateFiles.Strip();
3894 fTerminateFiles.ReplaceAll(" ",",");
3895 }
a2f5fc01 3896 TString outStream = "";
3897 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3898 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3899 ofstream out;
0d5d317c 3900 out.open(validationScript, ios::out);
c57f56b7 3901 out << "#!/bin/bash" << endl;
3902 out << "##################################################" << endl;
3903 out << "validateout=`dirname $0`" << endl;
3904 out << "validatetime=`date`" << endl;
3905 out << "validated=\"0\";" << endl;
3906 out << "error=0" << endl;
3907 out << "if [ -z $validateout ]" << endl;
3908 out << "then" << endl;
3909 out << " validateout=\".\"" << endl;
3910 out << "fi" << endl << endl;
3911 out << "cd $validateout;" << endl;
3912 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3913 out << "echo \"*******************************************************\"" << outStream << endl;
3914 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3915 out << "" << endl;
a2f5fc01 3916 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3917 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3918 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3919 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3920 out << "ls -la ./" << outStream << endl;
3921 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3922 out << "##################################################" << endl;
ebec370a 3923 out << "" << endl;
3924
3925 out << "if [ ! -f stderr ] ; then" << endl;
3926 out << " error=1" << endl;
a2f5fc01 3927 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3928 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3929 out << "fi" << endl;
3930
b34c9f51 3931 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3932 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3933 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3934 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3935 out << "" << endl;
3936
ebec370a 3937 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3938 out << " error=1" << endl;
a2f5fc01 3939 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3940 out << " echo \"$parArch\" " << outStream << endl;
3941 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3942 out << "fi" << endl;
3943
3944 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3945 out << " error=1" << endl;
a2f5fc01 3946 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3947 out << " echo \"$segViol\" " << outStream << endl;
3948 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3949 out << "fi" << endl;
3950
3951 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3952 out << " error=1" << endl;
a2f5fc01 3953 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3954 out << " echo \"$segFault\" " << outStream << endl;
3955 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3956 out << "fi" << endl;
3957
b34c9f51 3958 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3959 out << " error=1" << endl;
a2f5fc01 3960 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3961 out << " echo \"$glibcErr\" " << outStream << endl;
3962 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3963 out << "fi" << endl;
3964
ebec370a 3965 // Part dedicated to the specific analyses running into the train
3966
d3b18c4c 3967 TString outputFiles = fOutputFiles;
3968 if (merge && !fTerminateFiles.IsNull()) {
3969 outputFiles += ",";
3970 outputFiles += fTerminateFiles;
3971 }
3972 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3973 TIter next1(arr);
a2f5fc01 3974 TString outputFile;
bbd88db8 3975 while (!merge && (os=(TObjString*)next1())) {
3976 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3977 outputFile = os->GetString();
3978 Int_t index = outputFile.Index("@");
3979 if (index > 0) outputFile.Remove(index);
bbd88db8 3980 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3981 if (outputFile.Contains("*")) continue;
3982 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3983 out << " error=1" << endl;
7c2cd90a 3984 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3985 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3986 out << "fi" << endl;
3987 }
3988 delete arr;
7c2cd90a 3989 out << "if ! [ -f outputs_valid ] ; then" << endl;
3990 out << " error=1" << endl;
3991 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3992 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3993 out << "fi" << endl;
923e2ca5 3994
c57f56b7 3995 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3996 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3997 if (!IsKeepLogs()) {
3998 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3999 outStream = "";
149d288c 4000 out << " rm -f std*" << endl;
4001 }
c57f56b7 4002 out << "fi" << endl;
4003
a2f5fc01 4004 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
4005 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 4006 out << "cd -" << endl;
4007 out << "exit $error" << endl;
4008 }
4009 Bool_t copy = kTRUE;
d3b18c4c 4010 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 4011 if (copy) {
4012 CdWork();
4013 TString workdir = gGrid->GetHomeDirectory();
4014 workdir += fGridWorkingDir;
d3b18c4c 4015 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4016 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4017 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4018 }
4019}