]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Supressed warning when a data slot is not connected to a container
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
b3e07543 61 fMaxMergeStages(0),
d2a409b2 62 fNsubmitted(0),
a3e84053 63 fProductionMode(0),
cd11251e 64 fOutputToRunNo(0),
0f389141 65 fMergeViaJDL(0),
66 fFastReadOption(0),
e1c22e21 67 fOverwriteMode(1),
149d288c 68 fNreplicas(2),
3bdcb562 69 fNproofWorkers(0),
27734f0e 70 fNproofWorkersPerSlave(0),
3bdcb562 71 fProofReset(0),
c57f56b7 72 fRunNumbers(),
73 fExecutable(),
0a1c1f7f 74 fExecutableCommand(),
c57f56b7 75 fArguments(),
631c0b05 76 fExecutableArgs(),
c57f56b7 77 fAnalysisMacro(),
78 fAnalysisSource(),
d3b18c4c 79 fValidationScript(),
d5c6455a 80 fAdditionalRootLibs(),
c57f56b7 81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
648174cf 86 fExternalPackages(),
c57f56b7 87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
e7c71df0 95 fDatasetName(),
c57f56b7 96 fJDLName(),
d3b18c4c 97 fTerminateFiles(),
bb885a9e 98 fMergeExcludes(),
f965131e 99 fIncludePath(),
bb885a9e 100 fCloseSE(),
0df6ccf2 101 fFriendChainName(),
c6cb3634 102 fJobTag(),
648174cf 103 fOutputSingle(),
5fce53f4 104 fRunPrefix(),
3bdcb562 105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
f47d5cb4 110 fMergeDirName(),
4e5c5506 111 fInputFiles(0),
112 fPackages(0)
c57f56b7 113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
0f389141 122 fMergingJDL(NULL),
c57f56b7 123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
bb885a9e 128 fNtestFiles(0),
319593fb 129 fNrunsPerMaster(0),
16a4353c 130 fMaxMergeFiles(0),
b3e07543 131 fMaxMergeStages(0),
d2a409b2 132 fNsubmitted(0),
a3e84053 133 fProductionMode(0),
cd11251e 134 fOutputToRunNo(0),
0f389141 135 fMergeViaJDL(0),
136 fFastReadOption(0),
e1c22e21 137 fOverwriteMode(1),
149d288c 138 fNreplicas(2),
3bdcb562 139 fNproofWorkers(0),
27734f0e 140 fNproofWorkersPerSlave(0),
3bdcb562 141 fProofReset(0),
c57f56b7 142 fRunNumbers(),
143 fExecutable(),
0a1c1f7f 144 fExecutableCommand(),
c57f56b7 145 fArguments(),
631c0b05 146 fExecutableArgs(),
c57f56b7 147 fAnalysisMacro(),
148 fAnalysisSource(),
d3b18c4c 149 fValidationScript(),
d5c6455a 150 fAdditionalRootLibs(),
c57f56b7 151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
648174cf 156 fExternalPackages(),
c57f56b7 157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
e7c71df0 165 fDatasetName(),
c57f56b7 166 fJDLName(),
d3b18c4c 167 fTerminateFiles(),
bb885a9e 168 fMergeExcludes(),
f965131e 169 fIncludePath(),
bb885a9e 170 fCloseSE(),
0df6ccf2 171 fFriendChainName(),
c6cb3634 172 fJobTag(),
648174cf 173 fOutputSingle(),
5fce53f4 174 fRunPrefix(),
3bdcb562 175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
f47d5cb4 180 fMergeDirName(),
4e5c5506 181 fInputFiles(0),
182 fPackages(0)
c57f56b7 183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
0f389141 192 fMergingJDL(NULL),
c57f56b7 193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 198 fNtestFiles(other.fNtestFiles),
319593fb 199 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 200 fMaxMergeFiles(other.fMaxMergeFiles),
b3e07543 201 fMaxMergeStages(other.fMaxMergeStages),
d2a409b2 202 fNsubmitted(other.fNsubmitted),
a3e84053 203 fProductionMode(other.fProductionMode),
cd11251e 204 fOutputToRunNo(other.fOutputToRunNo),
0f389141 205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
149d288c 208 fNreplicas(other.fNreplicas),
3bdcb562 209 fNproofWorkers(other.fNproofWorkers),
27734f0e 210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 211 fProofReset(other.fProofReset),
c57f56b7 212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
0a1c1f7f 214 fExecutableCommand(other.fExecutableCommand),
c57f56b7 215 fArguments(other.fArguments),
631c0b05 216 fExecutableArgs(other.fExecutableArgs),
c57f56b7 217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 219 fValidationScript(other.fValidationScript),
d5c6455a 220 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
648174cf 226 fExternalPackages(other.fExternalPackages),
c57f56b7 227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
e7c71df0 235 fDatasetName(other.fDatasetName),
c57f56b7 236 fJDLName(other.fJDLName),
d3b18c4c 237 fTerminateFiles(other.fTerminateFiles),
bb885a9e 238 fMergeExcludes(other.fMergeExcludes),
f965131e 239 fIncludePath(other.fIncludePath),
bb885a9e 240 fCloseSE(other.fCloseSE),
0df6ccf2 241 fFriendChainName(other.fFriendChainName),
c6cb3634 242 fJobTag(other.fJobTag),
648174cf 243 fOutputSingle(other.fOutputSingle),
5fce53f4 244 fRunPrefix(other.fRunPrefix),
3bdcb562 245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
f47d5cb4 250 fMergeDirName(other.fMergeDirName),
4e5c5506 251 fInputFiles(0),
252 fPackages(0)
c57f56b7 253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
c57f56b7 259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
4e5c5506 266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
c57f56b7 273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
0f389141 280 if (fMergingJDL) delete fMergingJDL;
c57f56b7 281 if (fInputFiles) delete fInputFiles;
4e5c5506 282 if (fPackages) delete fPackages;
c57f56b7 283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 298 fNtestFiles = other.fNtestFiles;
a3e84053 299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
b3e07543 301 fMaxMergeStages = other.fMaxMergeStages;
a3e84053 302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
cd11251e 304 fOutputToRunNo = other.fOutputToRunNo;
0f389141 305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
149d288c 308 fNreplicas = other.fNreplicas;
3bdcb562 309 fNproofWorkers = other.fNproofWorkers;
27734f0e 310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 311 fProofReset = other.fProofReset;
c57f56b7 312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
0a1c1f7f 314 fExecutableCommand = other.fExecutableCommand;
c57f56b7 315 fArguments = other.fArguments;
631c0b05 316 fExecutableArgs = other.fExecutableArgs;
c57f56b7 317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 319 fValidationScript = other.fValidationScript;
d5c6455a 320 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
648174cf 326 fExternalPackages = other.fExternalPackages;
c57f56b7 327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
e7c71df0 335 fDatasetName = other.fDatasetName;
c57f56b7 336 fJDLName = other.fJDLName;
d3b18c4c 337 fTerminateFiles = other.fTerminateFiles;
bb885a9e 338 fMergeExcludes = other.fMergeExcludes;
f965131e 339 fIncludePath = other.fIncludePath;
bb885a9e 340 fCloseSE = other.fCloseSE;
0df6ccf2 341 fFriendChainName = other.fFriendChainName;
c6cb3634 342 fJobTag = other.fJobTag;
648174cf 343 fOutputSingle = other.fOutputSingle;
5fce53f4 344 fRunPrefix = other.fRunPrefix;
3bdcb562 345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
f47d5cb4 350 fMergeDirName = other.fMergeDirName;
c57f56b7 351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
4e5c5506 358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
c57f56b7 365 }
366 return *this;
367}
368
74013535 369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
f965131e 377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
74013535 391 fRunNumbers += Form(fRunPrefix.Data(), run);
c57f56b7 392}
393
98690895 394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunList(const char* runList)
396{
397// Add several runs into the list of runs; they are expected to be separated by a blank character.
398 TString sList = runList;
399 TObjArray *list = sList.Tokenize(" ");
400 Int_t n = list->GetEntries();
401 for (Int_t i = 0; i < n; i++) {
402 TObjString *os = (TObjString*)list->At(i);
403 AddRunNumber(os->GetString().Atoi());
404 }
405 delete list;
406}
407
ee75cfc3 408//______________________________________________________________________________
409void AliAnalysisAlien::AddRunNumber(const char* run)
410{
411// Add a run number to the list of runs to be processed.
412 if (fRunNumbers.Length()) fRunNumbers += " ";
413 fRunNumbers += run;
414}
415
c57f56b7 416//______________________________________________________________________________
417void AliAnalysisAlien::AddDataFile(const char *lfn)
418{
419// Adds a data file to the input to be analysed. The file should be a valid LFN
420// or point to an existing file in the alien workdir.
421 if (!fInputFiles) fInputFiles = new TObjArray();
422 fInputFiles->Add(new TObjString(lfn));
423}
648174cf 424
425//______________________________________________________________________________
426void AliAnalysisAlien::AddExternalPackage(const char *package)
427{
428// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
429 if (fExternalPackages) fExternalPackages += " ";
430 fExternalPackages += package;
431}
432
c57f56b7 433//______________________________________________________________________________
434Bool_t AliAnalysisAlien::Connect()
435{
436// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
437 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 438 if (fProductionMode) return kTRUE;
c57f56b7 439 if (!gGrid) {
440 Info("Connect", "Trying to connect to AliEn ...");
441 TGrid::Connect("alien://");
442 }
443 if (!gGrid || !gGrid->IsConnected()) {
444 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
445 return kFALSE;
446 }
447 fUser = gGrid->GetUser();
448 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
449 return kTRUE;
450}
451
452//______________________________________________________________________________
453void AliAnalysisAlien::CdWork()
454{
455// Check validity of alien workspace. Create directory if possible.
456 if (!Connect()) {
457 Error("CdWork", "Alien connection required");
458 return;
459 }
460 TString homedir = gGrid->GetHomeDirectory();
461 TString workdir = homedir + fGridWorkingDir;
923e2ca5 462 if (DirectoryExists(workdir)) {
463 gGrid->Cd(workdir);
464 return;
465 }
466 // Work directory not existing - create it
467 gGrid->Cd(homedir);
b93f8109 468 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 469 gGrid->Cd(fGridWorkingDir);
d3b18c4c 470 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 471 } else {
d3b18c4c 472 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 473 workdir.Data(), homedir.Data());
474 fGridWorkingDir = "";
475 }
c57f56b7 476}
477
348be253 478//______________________________________________________________________________
479Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
480{
481// Check if file copying is possible.
d3b18c4c 482 if (fProductionMode) return kTRUE;
348be253 483 if (!Connect()) {
484 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
485 return kFALSE;
486 }
d3339be3 487 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
488 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 489 // Check if alien_CLOSE_SE is defined
490 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
491 if (!closeSE.IsNull()) {
492 Info("CheckFileCopy", "Your current close storage is pointing to: \
493 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
494 } else {
495 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
496 }
497 // Check if grid directory exists.
498 if (!DirectoryExists(alienpath)) {
499 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
500 return kFALSE;
501 }
d3339be3 502 TFile f("plugin_test_copy", "RECREATE");
348be253 503 // User may not have write permissions to current directory
504 if (f.IsZombie()) {
505 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
506 gSystem->WorkingDirectory());
507 return kFALSE;
508 }
509 f.Close();
d3339be3 510 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
511 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
512 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 513 \n# 1. Make sure you have write permissions there. If this is the case: \
514 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
515 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
516 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
517 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
518 gSystem->Unlink(f.GetName());
519 return kFALSE;
520 }
521 gSystem->Unlink(f.GetName());
d3339be3 522 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
523 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 524 return kTRUE;
525}
526
c57f56b7 527//______________________________________________________________________________
528Bool_t AliAnalysisAlien::CheckInputData()
529{
530// Check validity of input data. If necessary, create xml files.
d3b18c4c 531 if (fProductionMode) return kTRUE;
a8739e8a 532 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 533 if (!fGridDataDir.Length()) {
534 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
535 return kFALSE;
536 }
9f020952 537 if (fMergeViaJDL) {
538 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
539 return kFALSE;
540 }
d2a409b2 541 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 542 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
543 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 544 return kTRUE;
c57f56b7 545 }
546 // Process declared files
a2f5fc01 547 Bool_t isCollection = kFALSE;
548 Bool_t isXml = kFALSE;
549 Bool_t useTags = kFALSE;
c57f56b7 550 Bool_t checked = kFALSE;
d3b18c4c 551 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 552 TString file;
553 TString workdir = gGrid->GetHomeDirectory();
554 workdir += fGridWorkingDir;
555 if (fInputFiles) {
556 TObjString *objstr;
557 TIter next(fInputFiles);
558 while ((objstr=(TObjString*)next())) {
559 file = workdir;
560 file += "/";
561 file += objstr->GetString();
562 // Store full lfn path
563 if (FileExists(file)) objstr->SetString(file);
564 else {
565 file = objstr->GetName();
566 if (!FileExists(objstr->GetName())) {
567 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
568 objstr->GetName(), workdir.Data());
569 return kFALSE;
570 }
571 }
572 Bool_t iscoll, isxml, usetags;
573 CheckDataType(file, iscoll, isxml, usetags);
574 if (!checked) {
575 checked = kTRUE;
a2f5fc01 576 isCollection = iscoll;
577 isXml = isxml;
578 useTags = usetags;
579 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 580 } else {
a2f5fc01 581 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 582 Error("CheckInputData", "Some conflict was found in the types of inputs");
583 return kFALSE;
584 }
585 }
586 }
587 }
588 // Process requested run numbers
a8739e8a 589 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 590 // Check validity of alien data directory
591 if (!fGridDataDir.Length()) {
592 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
593 return kFALSE;
594 }
923e2ca5 595 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 596 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
597 return kFALSE;
598 }
a2f5fc01 599 if (isCollection) {
c57f56b7 600 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
601 return kFALSE;
602 }
603
a2f5fc01 604 if (checked && !isXml) {
c57f56b7 605 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
606 return kFALSE;
607 }
608 // Check validity of run number(s)
609 TObjArray *arr;
610 TObjString *os;
74013535 611 TString format;
319593fb 612 Int_t nruns = 0;
904f9f5f 613 TString schunk, schunk2;
c57f56b7 614 TString path;
615 if (!checked) {
616 checked = kTRUE;
a2f5fc01 617 useTags = fDataPattern.Contains("tag");
618 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 619 }
a2f5fc01 620 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 621 Error("CheckInputData", "Cannot mix input files using/not using tags");
622 return kFALSE;
623 }
624 if (fRunNumbers.Length()) {
a8739e8a 625 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 626 arr = fRunNumbers.Tokenize(" ");
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
629 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 630 if (!DirectoryExists(path)) {
631 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 632 continue;
c57f56b7 633 }
634 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
635 TString msg = "\n##### file: ";
636 msg += path;
637 msg += " type: xml_collection;";
a2f5fc01 638 if (useTags) msg += " using_tags: Yes";
c57f56b7 639 else msg += " using_tags: No";
23329835 640 Info("CheckDataType", "%s", msg.Data());
319593fb 641 if (fNrunsPerMaster<2) {
d2a409b2 642 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 643 } else {
644 nruns++;
645 if (((nruns-1)%fNrunsPerMaster) == 0) {
646 schunk = os->GetString();
647 }
648 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
649 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 650 AddDataFile(schunk);
319593fb 651 }
c57f56b7 652 }
653 delete arr;
a8739e8a 654 } else {
655 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
656 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 657 format = Form("%%s/%s ", fRunPrefix.Data());
658 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 659 if (!DirectoryExists(path)) {
a8739e8a 660 continue;
661 }
74013535 662 format = Form("%%s/%s.xml", fRunPrefix.Data());
663 path = Form(format.Data(), workdir.Data(),irun);
a8739e8a 664 TString msg = "\n##### file: ";
665 msg += path;
666 msg += " type: xml_collection;";
a2f5fc01 667 if (useTags) msg += " using_tags: Yes";
a8739e8a 668 else msg += " using_tags: No";
23329835 669 Info("CheckDataType", "%s", msg.Data());
319593fb 670 if (fNrunsPerMaster<2) {
74013535 671 format = Form("%s.xml", fRunPrefix.Data());
672 AddDataFile(Form(format.Data(),irun));
319593fb 673 } else {
674 nruns++;
675 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 676 schunk = Form(fRunPrefix.Data(),irun);
319593fb 677 }
74013535 678 format = Form("_%s.xml", fRunPrefix.Data());
679 schunk2 = Form(format.Data(), irun);
319593fb 680 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 681 schunk += schunk2;
d2a409b2 682 AddDataFile(schunk);
319593fb 683 }
a8739e8a 684 }
904f9f5f 685 if (!fInputFiles) {
686 schunk += schunk2;
687 AddDataFile(schunk);
688 }
c57f56b7 689 }
690 return kTRUE;
691}
692
693//______________________________________________________________________________
694Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
695{
696// Create dataset for the grid data directory + run number.
d3b18c4c 697 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 698 if (!Connect()) {
699 Error("CreateDataset", "Cannot create dataset with no grid connection");
700 return kFALSE;
701 }
702
703 // Cd workspace
d3b18c4c 704 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 705 TString workdir = gGrid->GetHomeDirectory();
706 workdir += fGridWorkingDir;
707
708 // Compose the 'find' command arguments
74013535 709 TString format;
c57f56b7 710 TString command;
711 TString options = "-x collection ";
bb885a9e 712 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 713 TString conditions = "";
714
715 TString file;
716 TString path;
319593fb 717 Int_t nruns = 0;
904f9f5f 718 TString schunk, schunk2;
ab254fd1 719 TGridCollection *cbase=0, *cadd=0;
d2a409b2 720 if (!fRunNumbers.Length() && !fRunRange[0]) {
721 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
722 // Make a single data collection from data directory.
723 path = fGridDataDir;
923e2ca5 724 if (!DirectoryExists(path)) {
d2a409b2 725 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
726 return kFALSE;
727 }
923e2ca5 728// CdWork();
d2a409b2 729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 731 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 732 command = "find ";
733 command += options;
734 command += path;
735 command += " ";
736 command += pattern;
737 command += conditions;
84fcd93f 738 printf("command: %s\n", command.Data());
d2a409b2 739 TGridResult *res = gGrid->Command(command);
740 if (res) delete res;
741 // Write standard output to file
742 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 743 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
744 Bool_t nullFile = kFALSE;
745 if (!hasGrep) {
d3339be3 746 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
747 } else {
a2f5fc01 748 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
749 if (nullFile) {
defd7a3a 750 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
751 return kFALSE;
752 }
d3339be3 753 }
0f389141 754 }
755 Bool_t fileExists = FileExists(file);
756 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 757 // Copy xml file to alien space
0f389141 758 if (fileExists) gGrid->Rm(file);
d2a409b2 759 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
760 if (!FileExists(file)) {
761 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
762 return kFALSE;
763 }
764 // Update list of files to be processed.
765 }
766 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
767 return kTRUE;
768 }
c57f56b7 769 // Several runs
a2f5fc01 770 Bool_t nullResult = kTRUE;
a8739e8a 771 if (fRunNumbers.Length()) {
772 TObjArray *arr = fRunNumbers.Tokenize(" ");
773 TObjString *os;
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 777 if (!DirectoryExists(path)) continue;
778// CdWork();
a8739e8a 779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
780 else file = Form("%s.xml", os->GetString().Data());
319593fb 781 // If local collection file does not exist, create it via 'find' command.
d3339be3 782 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 783 command = "find ";
784 command += options;
785 command += path;
786 command += pattern;
787 command += conditions;
788 TGridResult *res = gGrid->Command(command);
789 if (res) delete res;
790 // Write standard output to file
791 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 792 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
793 Bool_t nullFile = kFALSE;
794 if (!hasGrep) {
d3339be3 795 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
796 } else {
a2f5fc01 797 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
798 if (nullFile) {
defd7a3a 799 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
800 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
801 continue;
802 }
d3339be3 803 }
a2f5fc01 804 nullResult = kFALSE;
d3339be3 805 }
a8739e8a 806 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 807 // Check if there is one run per master job.
808 if (fNrunsPerMaster<2) {
809 if (FileExists(file)) {
0f389141 810 if (fOverwriteMode) gGrid->Rm(file);
811 else {
812 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
813 continue;
814 }
319593fb 815 }
816 // Copy xml file to alien space
817 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
818 if (!FileExists(file)) {
819 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
820 delete arr;
821 return kFALSE;
822 }
823 } else {
824 nruns++;
825 if (((nruns-1)%fNrunsPerMaster) == 0) {
826 schunk = os->GetString();
827 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
828 } else {
829 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 830 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 831 cbase->Add(cadd);
832 delete cadd;
833 }
834 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
835 continue;
836 }
837 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 838 if (FileExists(schunk)) {
839 if (fOverwriteMode) gGrid->Rm(file);
840 else {
841 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
842 continue;
843 }
319593fb 844 }
84fcd93f 845 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 846 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 847 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 848 if (!FileExists(schunk)) {
849 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
850 delete arr;
851 return kFALSE;
852 }
d3339be3 853 }
a8739e8a 854 }
855 delete arr;
a2f5fc01 856 if (nullResult) {
d3339be3 857 Error("CreateDataset", "No valid dataset corresponding to the query!");
858 return kFALSE;
859 }
a8739e8a 860 } else {
861 // Process a full run range.
862 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
74013535 863 format = Form("%%s/%s ", fRunPrefix.Data());
864 path = Form(format.Data(), fGridDataDir.Data(), irun);
923e2ca5 865 if (!DirectoryExists(path)) continue;
866// CdWork();
74013535 867 format = Form("%s.xml", fRunPrefix.Data());
a8739e8a 868 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
74013535 869 else file = Form(format.Data(), irun);
0f389141 870 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
871 if (fOverwriteMode) gGrid->Rm(file);
872 else {
873 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
874 continue;
875 }
a8739e8a 876 }
319593fb 877 // If local collection file does not exist, create it via 'find' command.
d3339be3 878 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 879 command = "find ";
880 command += options;
881 command += path;
882 command += pattern;
883 command += conditions;
884 TGridResult *res = gGrid->Command(command);
885 if (res) delete res;
886 // Write standard output to file
887 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 888 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
889 Bool_t nullFile = kFALSE;
890 if (!hasGrep) {
d3339be3 891 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
892 } else {
a2f5fc01 893 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
894 if (nullFile) {
defd7a3a 895 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
896 continue;
897 }
d3339be3 898 }
a2f5fc01 899 nullResult = kFALSE;
319593fb 900 }
a8739e8a 901 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 902 // Check if there is one run per master job.
903 if (fNrunsPerMaster<2) {
904 if (FileExists(file)) {
0f389141 905 if (fOverwriteMode) gGrid->Rm(file);
906 else {
907 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
908 continue;
909 }
319593fb 910 }
911 // Copy xml file to alien space
912 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
913 if (!FileExists(file)) {
914 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
915 return kFALSE;
916 }
917 } else {
918 nruns++;
95e5b448 919 // Check if the collection for the chunk exist locally.
920 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 921 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
922 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
923 else continue;
924 }
84fcd93f 925 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 926 if (((nruns-1)%fNrunsPerMaster) == 0) {
74013535 927 schunk = Form(fRunPrefix.Data(), irun);
319593fb 928 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
929 } else {
930 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 cbase->Add(cadd);
932 delete cadd;
933 }
74013535 934 format = Form("%%s_%s.xml", fRunPrefix.Data());
935 schunk2 = Form(format.Data(), schunk.Data(), irun);
904f9f5f 936 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 937 continue;
938 }
904f9f5f 939 schunk = schunk2;
319593fb 940 if (FileExists(schunk)) {
0f389141 941 if (fOverwriteMode) gGrid->Rm(schunk);
942 else {
943 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
944 continue;
945 }
319593fb 946 }
84fcd93f 947 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 948 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 949 if (FileExists(schunk)) {
0f389141 950 if (fOverwriteMode) gGrid->Rm(schunk);
951 else {
952 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
953 continue;
954 }
95e5b448 955 }
319593fb 956 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
957 if (!FileExists(schunk)) {
958 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
959 return kFALSE;
960 }
961 }
c57f56b7 962 }
a2f5fc01 963 if (nullResult) {
d3339be3 964 Error("CreateDataset", "No valid dataset corresponding to the query!");
965 return kFALSE;
966 }
a8739e8a 967 }
c57f56b7 968 return kTRUE;
969}
970
971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CreateJDL()
973{
974// Generate a JDL file according to current settings. The name of the file is
975// specified by fJDLName.
976 Bool_t error = kFALSE;
977 TObjArray *arr = 0;
978 Bool_t copy = kTRUE;
d3b18c4c 979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 980 Bool_t generate = kTRUE;
981 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
982 if (!Connect()) {
983 Error("CreateJDL", "Alien connection required");
984 return kFALSE;
985 }
986 // Check validity of alien workspace
d3b18c4c 987 TString workdir;
988 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
989 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 990 workdir += fGridWorkingDir;
991 if (generate) {
992 TObjString *os;
993 if (!fInputFiles) {
994 Error("CreateJDL()", "Define some input files for your analysis.");
995 error = kTRUE;
996 }
997 // Compose list of input files
998 // Check if output files were defined
999 if (!fOutputFiles.Length()) {
1000 Error("CreateJDL", "You must define at least one output file");
1001 error = kTRUE;
1002 }
1003 // Check if an output directory was defined and valid
1004 if (!fGridOutputDir.Length()) {
1005 Error("CreateJDL", "You must define AliEn output directory");
1006 error = kTRUE;
1007 } else {
d3b18c4c 1008 if (!fProductionMode) {
1009 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1010 if (!DirectoryExists(fGridOutputDir)) {
1011 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1012 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1013 } else {
1014 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1015 // error = kTRUE;
1016 }
67a3b254 1017 } else {
1018 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
d3b18c4c 1019 }
1020 gGrid->Cd(workdir);
1021 }
c57f56b7 1022 }
1023 // Exit if any error up to now
1024 if (error) return kFALSE;
1025 // Set JDL fields
0f389141 1026 if (!fUser.IsNull()) {
1027 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1028 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1029 }
1030 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1031 TString mergeExec = fExecutable;
1032 mergeExec.ReplaceAll(".sh", "_merge.sh");
1033 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1034 mergeExec.ReplaceAll(".sh", ".C");
1035 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 1036 if (!fArguments.IsNull())
1037 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 1038 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
f47d5cb4 1039 else {
1040 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1041 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1042 }
b3e07543 1043
1f0d1ca2 1044 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1045 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1046 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048
0f389141 1049 if (fMaxInitFailed > 0) {
c57f56b7 1050 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1051 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1052 }
1053 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1054 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1055 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
b3e07543 1056 }
1057 if (!IsOneStageMerging()) {
f47d5cb4 1058 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
b3e07543 1059 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
0f389141 1060 }
1061 if (fSplitMode.Length()) {
c57f56b7 1062 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1063 fGridJDL->SetDescription("Split", "We split per SE or file");
b3e07543 1064 }
1065 fMergingJDL->SetValue("Split", "\"se\"");
1066 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
0f389141 1067 if (!fAliROOTVersion.IsNull()) {
1068 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1069 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1070 }
1071 if (!fROOTVersion.IsNull()) {
c57f56b7 1072 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1073 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1074 }
1075 if (!fAPIVersion.IsNull()) {
c57f56b7 1076 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1077 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1078 }
648174cf 1079 if (!fExternalPackages.IsNull()) {
1080 arr = fExternalPackages.Tokenize(" ");
1081 TIter next(arr);
1082 while ((os=(TObjString*)next())) {
1083 TString pkgname = os->GetString();
1084 Int_t index = pkgname.Index("::");
1085 TString pkgversion = pkgname(index+2, pkgname.Length());
1086 pkgname.Remove(index);
1087 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1088 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1089 }
1090 delete arr;
1091 }
0f389141 1092 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1093 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
b3e07543 1094 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
0f389141 1096 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1097 TString analysisFile = fExecutable;
1098 analysisFile.ReplaceAll(".sh", ".root");
1099 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1100 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1101 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1102 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1103 if (fAdditionalLibs.Length()) {
1104 arr = fAdditionalLibs.Tokenize(" ");
1105 TIter next(arr);
1106 while ((os=(TObjString*)next())) {
1107 if (os->GetString().Contains(".so")) continue;
1108 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1109 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1110 }
1111 delete arr;
1112 }
4e5c5506 1113 if (fPackages) {
1114 TIter next(fPackages);
1115 TObject *obj;
0f389141 1116 while ((obj=next())) {
4e5c5506 1117 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1118 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1119 }
4e5c5506 1120 }
c57f56b7 1121 if (fOutputArchive.Length()) {
1122 arr = fOutputArchive.Tokenize(" ");
1123 TIter next(arr);
0f389141 1124 Bool_t first = kTRUE;
1125 const char *comment = "Files to be archived";
1126 const char *comment1 = comment;
1127 while ((os=(TObjString*)next())) {
1128 if (!first) comment = NULL;
1129 if (!os->GetString().Contains("@") && fCloseSE.Length())
1130 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1131 else
1132 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1133 first = kFALSE;
1134 }
c57f56b7 1135 delete arr;
f790bc1b 1136 // Output archive for the merging jdl
1137 TString outputArchive;
1138 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
b83f84f0 1139 outputArchive = "log_archive.zip:std*@disk=1 ";
f790bc1b 1140 // Add normal output files, extra files + terminate files
1141 TString files = GetListOfFiles("outextter");
1142 // Do not register merge excludes
1143 if (!fMergeExcludes.IsNull()) {
1144 arr = fMergeExcludes.Tokenize(" ");
1145 TIter next1(arr);
1146 while ((os=(TObjString*)next1())) {
1147 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1148 files.ReplaceAll(os->GetString(),"");
1149 }
66d71516 1150 delete arr;
0f389141 1151 }
f790bc1b 1152 files.ReplaceAll(".root", "*.root");
b83f84f0 1153 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
f790bc1b 1154 } else {
95f84779 1155 TString files = fOutputArchive;
1156 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1157 outputArchive = files;
f790bc1b 1158 }
0f389141 1159 arr = outputArchive.Tokenize(" ");
705adb3e 1160 TIter next2(arr);
0f389141 1161 comment = comment1;
1162 first = kTRUE;
705adb3e 1163 while ((os=(TObjString*)next2())) {
0f389141 1164 if (!first) comment = NULL;
0cdf65a8 1165 TString currentfile = os->GetString();
0cdf65a8 1166 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1167 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1168 else
0cdf65a8 1169 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1170 first = kFALSE;
1171 }
1172 delete arr;
c57f56b7 1173 }
149d288c 1174 arr = fOutputFiles.Tokenize(",");
c57f56b7 1175 TIter next(arr);
0f389141 1176 Bool_t first = kTRUE;
f790bc1b 1177 const char *comment = "Files to be saved";
43da816a 1178 while ((os=(TObjString*)next())) {
e1eaf596 1179 // Ignore ouputs in jdl that are also in outputarchive
1180 TString sout = os->GetString();
f790bc1b 1181 sout.ReplaceAll("*", "");
1182 sout.ReplaceAll(".root", "");
e1eaf596 1183 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1184 if (fOutputArchive.Contains(sout)) continue;
0f389141 1185 if (!first) comment = NULL;
43da816a 1186 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1187 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1188 else
0f389141 1189 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
b3e07543 1190 first = kFALSE;
f790bc1b 1191 if (fMergeExcludes.Contains(sout)) continue;
1192 if (!os->GetString().Contains("@") && fCloseSE.Length())
1193 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1194 else
1195 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
43da816a 1196 }
c57f56b7 1197 delete arr;
0f389141 1198 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1199 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1200 TString validationScript = fValidationScript;
0f389141 1201 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1202 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1203 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1204 if (fMasterResubmitThreshold) {
1205 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1206 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1207 }
d2a409b2 1208 // Write a jdl with 2 input parameters: collection name and output dir name.
1209 WriteJDL(copy);
c57f56b7 1210 }
1211 // Copy jdl to grid workspace
a8739e8a 1212 if (copy) {
b5e4aaa7 1213 // Check if an output directory was defined and valid
1214 if (!fGridOutputDir.Length()) {
1215 Error("CreateJDL", "You must define AliEn output directory");
1216 return kFALSE;
1217 } else {
1218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1219 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1220 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1221 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1222 } else {
1223 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 }
1227 gGrid->Cd(workdir);
1228 }
648174cf 1229 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1230 TString mergeJDLName = fExecutable;
1231 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1232 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1233 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1234 if (fProductionMode) {
648174cf 1235 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1236 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1237 }
648174cf 1238 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1239 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1240 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1241 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1242 if (fMergeViaJDL) {
1243 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1244 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1245 }
648174cf 1246 }
c57f56b7 1247 if (fAdditionalLibs.Length()) {
e7c71df0 1248 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
c57f56b7 1252 if (os->GetString().Contains(".so")) continue;
4e5c5506 1253 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1254 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1255 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1256 }
1257 delete arr;
1258 }
4e5c5506 1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 while ((obj=next())) {
fdbbc7be 1263 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1265 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1266 }
1267 }
c57f56b7 1268 }
1269 return kTRUE;
1270}
1271
a8739e8a 1272//______________________________________________________________________________
d2a409b2 1273Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1274{
1275// Writes one or more JDL's corresponding to findex. If findex is negative,
1276// all run numbers are considered in one go (jdl). For non-negative indices
1277// they correspond to the indices in the array fInputFiles.
1278 if (!fInputFiles) return kFALSE;
d3b18c4c 1279 TObject *os;
1280 TString workdir;
1281 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1282 workdir += fGridWorkingDir;
f47d5cb4 1283 TString stageName = "$2";
1284 if (fProductionMode) stageName = "$4";
1285 if (!fMergeDirName.IsNull()) {
1286 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1287 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1288 } else {
1289 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1290 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1291 }
d3b18c4c 1292 if (fProductionMode) {
a8739e8a 1293 TIter next(fInputFiles);
b3e07543 1294 while ((os=next())) {
d3b18c4c 1295 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
b3e07543 1296 }
d3b18c4c 1297 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
d3b18c4c 1298 } else {
1299 if (!fRunNumbers.Length() && !fRunRange[0]) {
1300 // One jdl with no parameters in case input data is specified by name.
1301 TIter next(fInputFiles);
1302 while ((os=next()))
1303 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1304 if (!fOutputSingle.IsNull())
1305 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1306 else {
1307 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1308 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1309 }
1310 } else {
1311 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1312 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1313 if (!fOutputSingle.IsNull()) {
1314 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1315 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1316 } else {
1317 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
d3b18c4c 1318 }
1319 }
a8739e8a 1320 }
1321
a8739e8a 1322 // Generate the JDL as a string
1323 TString sjdl = fGridJDL->Generate();
0f389141 1324 TString sjdl1 = fMergingJDL->Generate();
f47d5cb4 1325 // Final merge jdl
1326 if (!fMergeDirName.IsNull()) {
1327 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1328 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1329 } else {
1330 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1331 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1332 }
b3e07543 1333 TString sjdl2 = fMergingJDL->Generate();
1334 Int_t index, index1;
a8739e8a 1335 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1336 sjdl.ReplaceAll("(member", "\n (member");
1337 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1338 sjdl.ReplaceAll("{", "{\n ");
1339 sjdl.ReplaceAll("};", "\n};");
1340 sjdl.ReplaceAll("{\n \n", "{\n");
1341 sjdl.ReplaceAll("\n\n", "\n");
1342 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1343 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1344 sjdl1.ReplaceAll("(member", "\n (member");
1345 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1346 sjdl1.ReplaceAll("{", "{\n ");
1347 sjdl1.ReplaceAll("};", "\n};");
1348 sjdl1.ReplaceAll("{\n \n", "{\n");
1349 sjdl1.ReplaceAll("\n\n", "\n");
1350 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
b3e07543 1351 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1352 sjdl2.ReplaceAll("(member", "\n (member");
1353 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1354 sjdl2.ReplaceAll("{", "{\n ");
1355 sjdl2.ReplaceAll("};", "\n};");
1356 sjdl2.ReplaceAll("{\n \n", "{\n");
1357 sjdl2.ReplaceAll("\n\n", "\n");
1358 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1359 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1360 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1361 index = sjdl.Index("JDLVariables");
1362 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1363 sjdl += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1364 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
0f389141 1365 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1366 index = fJobTag.Index(":");
1367 if (index < 0) index = fJobTag.Length();
1368 TString jobTag = fJobTag;
f47d5cb4 1369 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
d3b18c4c 1370 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
f47d5cb4 1371 if (fProductionMode) {
1372 sjdl1.Prepend("# Generated merging jdl (production mode) \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = train number \
1375 \n# $3 = production (like LHC10b) \
1376 \n# $4 = merging stage \
1377 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1378 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1379 sjdl2.Prepend("# Generated merging jdl \
1380 \n# $1 = full alien path to output directory to be merged \
1381 \n# $2 = train number \
1382 \n# $3 = production (like LHC10b) \
1383 \n# $4 = merging stage \
1384 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1385 } else {
1386 sjdl1.Prepend("# Generated merging jdl \
1387 \n# $1 = full alien path to output directory to be merged \
1388 \n# $2 = merging stage \
03335324 1389 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1390 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1391 sjdl2.Prepend("# Generated merging jdl \
1392 \n# $1 = full alien path to output directory to be merged \
1393 \n# $2 = merging stage \
03335324 1394 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
f47d5cb4 1395 }
0f389141 1396 index = sjdl1.Index("JDLVariables");
1397 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
b3e07543 1398 index = sjdl2.Index("JDLVariables");
1399 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
4739b2aa 1400 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
b3e07543 1401 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1402 index = sjdl2.Index("Split =");
1403 if (index>=0) {
1404 index1 = sjdl2.Index("\n", index);
1405 sjdl2.Remove(index, index1-index+1);
1406 }
1407 index = sjdl2.Index("SplitMaxInputFileNumber");
1408 if (index>=0) {
1409 index1 = sjdl2.Index("\n", index);
1410 sjdl2.Remove(index, index1-index+1);
1411 }
1412 index = sjdl2.Index("InputDataCollection");
1413 if (index>=0) {
1414 index1 = sjdl2.Index(";", index);
1415 sjdl2.Remove(index, index1-index+1);
1416 }
1417 index = sjdl2.Index("InputDataListFormat");
1418 if (index>=0) {
1419 index1 = sjdl2.Index("\n", index);
1420 sjdl2.Remove(index, index1-index+1);
1421 }
1422 index = sjdl2.Index("InputDataList");
1423 if (index>=0) {
1424 index1 = sjdl2.Index("\n", index);
1425 sjdl2.Remove(index, index1-index+1);
1426 }
f47d5cb4 1427 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
a8739e8a 1428 // Write jdl to file
a8739e8a 1429 ofstream out;
d2a409b2 1430 out.open(fJDLName.Data(), ios::out);
a8739e8a 1431 if (out.bad()) {
d3b18c4c 1432 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1433 return kFALSE;
1434 }
1435 out << sjdl << endl;
b3e07543 1436 out.close();
0f389141 1437 TString mergeJDLName = fExecutable;
1438 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1439 if (fMergeViaJDL) {
1440 ofstream out1;
1441 out1.open(mergeJDLName.Data(), ios::out);
b3e07543 1442 if (out1.bad()) {
d3b18c4c 1443 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1444 return kFALSE;
1445 }
1446 out1 << sjdl1 << endl;
b3e07543 1447 out1.close();
1448 ofstream out2;
1449 TString finalJDL = mergeJDLName;
1450 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1451 out2.open(finalJDL.Data(), ios::out);
1452 if (out2.bad()) {
1453 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1454 return kFALSE;
1455 }
1456 out2 << sjdl2 << endl;
1457 out2.close();
0f389141 1458 }
a8739e8a 1459
1460 // Copy jdl to grid workspace
1461 if (!copy) {
d3b18c4c 1462 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1463 } else {
d2a409b2 1464 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1465 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
b3e07543 1466 TString finalJDL = mergeJDLName;
1467 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1468 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
0f389141 1469 if (fProductionMode) {
b5fe9cba 1470 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1471 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
b3e07543 1472 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
0f389141 1473 }
d2a409b2 1474 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1475 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
b3e07543 1476 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
d3b18c4c 1477 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1478 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1479 if (fMergeViaJDL) {
b3e07543 1480 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1481 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
b3e07543 1482 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
0f389141 1483 }
a8739e8a 1484 }
1485 return kTRUE;
1486}
1487
c57f56b7 1488//______________________________________________________________________________
5513444a 1489Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1490{
1491// Returns true if file exists.
5513444a 1492 if (!gGrid) return kFALSE;
2ea55496 1493 TString slfn = lfn;
1494 slfn.ReplaceAll("alien://","");
1495 TGridResult *res = gGrid->Ls(slfn);
c57f56b7 1496 if (!res) return kFALSE;
1497 TMap *map = dynamic_cast<TMap*>(res->At(0));
1498 if (!map) {
1499 delete res;
1500 return kFALSE;
1501 }
1502 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1503 if (!objs || !objs->GetString().Length()) {
1504 delete res;
1505 return kFALSE;
1506 }
1507 delete res;
1508 return kTRUE;
1509}
1510
923e2ca5 1511//______________________________________________________________________________
1512Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1513{
1514// Returns true if directory exists. Can be also a path.
1515 if (!gGrid) return kFALSE;
1516 // Check if dirname is a path
1517 TString dirstripped = dirname;
1518 dirstripped = dirstripped.Strip();
1519 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1520 TString dir = gSystem->BaseName(dirstripped);
1521 dir += "/";
1522 TString path = gSystem->DirName(dirstripped);
1523 TGridResult *res = gGrid->Ls(path, "-F");
1524 if (!res) return kFALSE;
1525 TIter next(res);
1526 TMap *map;
1527 TObject *obj;
1528 while ((map=dynamic_cast<TMap*>(next()))) {
1529 obj = map->GetValue("name");
1530 if (!obj) break;
1531 if (dir == obj->GetName()) {
1532 delete res;
1533 return kTRUE;
1534 }
1535 }
1536 delete res;
1537 return kFALSE;
1538}
1539
c57f56b7 1540//______________________________________________________________________________
a2f5fc01 1541void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1542{
1543// Check input data type.
a2f5fc01 1544 isCollection = kFALSE;
1545 isXml = kFALSE;
1546 useTags = kFALSE;
c57f56b7 1547 if (!gGrid) {
1548 Error("CheckDataType", "No connection to grid");
1549 return;
1550 }
a2f5fc01 1551 isCollection = IsCollection(lfn);
c57f56b7 1552 TString msg = "\n##### file: ";
1553 msg += lfn;
a2f5fc01 1554 if (isCollection) {
c57f56b7 1555 msg += " type: raw_collection;";
1556 // special treatment for collections
a2f5fc01 1557 isXml = kFALSE;
c57f56b7 1558 // check for tag files in the collection
1559 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1560 if (!res) {
1561 msg += " using_tags: No (unknown)";
23329835 1562 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1563 return;
1564 }
1565 const char* typeStr = res->GetKey(0, "origLFN");
1566 if (!typeStr || !strlen(typeStr)) {
1567 msg += " using_tags: No (unknown)";
23329835 1568 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1569 return;
1570 }
1571 TString file = typeStr;
a2f5fc01 1572 useTags = file.Contains(".tag");
1573 if (useTags) msg += " using_tags: Yes";
c57f56b7 1574 else msg += " using_tags: No";
23329835 1575 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1576 return;
1577 }
1578 TString slfn(lfn);
1579 slfn.ToLower();
a2f5fc01 1580 isXml = slfn.Contains(".xml");
1581 if (isXml) {
c57f56b7 1582 // Open xml collection and check if there are tag files inside
1583 msg += " type: xml_collection;";
1584 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1585 if (!coll) {
1586 msg += " using_tags: No (unknown)";
23329835 1587 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1588 return;
1589 }
1590 TMap *map = coll->Next();
1591 if (!map) {
1592 msg += " using_tags: No (unknown)";
23329835 1593 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1594 return;
1595 }
1596 map = (TMap*)map->GetValue("");
1597 TString file;
1598 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1599 useTags = file.Contains(".tag");
c57f56b7 1600 delete coll;
a2f5fc01 1601 if (useTags) msg += " using_tags: Yes";
c57f56b7 1602 else msg += " using_tags: No";
23329835 1603 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1604 return;
1605 }
a2f5fc01 1606 useTags = slfn.Contains(".tag");
c57f56b7 1607 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1608 else msg += " type: unknown file;";
a2f5fc01 1609 if (useTags) msg += " using_tags: Yes";
c57f56b7 1610 else msg += " using_tags: No";
23329835 1611 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1612}
1613
4e5c5506 1614//______________________________________________________________________________
1615void AliAnalysisAlien::EnablePackage(const char *package)
1616{
1617// Enables a par file supposed to exist in the current directory.
1618 TString pkg(package);
1619 pkg.ReplaceAll(".par", "");
1620 pkg += ".par";
1621 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1622 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1623 return;
1624 }
1625 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1626 Info("EnablePackage", "AliEn plugin will use .par packages");
1627 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1628 if (!fPackages) {
1629 fPackages = new TObjArray();
1630 fPackages->SetOwner();
1631 }
1632 fPackages->Add(new TObjString(pkg));
1633}
1634
3bdcb562 1635//______________________________________________________________________________
1636TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1637{
1638// Make a tree from files having the location specified in fFileForTestMode.
1639// Inspired from JF's CreateESDChain.
1640 if (fFileForTestMode.IsNull()) {
1641 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1642 return NULL;
1643 }
1644 if (gSystem->AccessPathName(fFileForTestMode)) {
1645 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1646 return NULL;
1647 }
1648 // Open the file
1649 ifstream in;
1650 in.open(fFileForTestMode);
1651 Int_t count = 0;
1652 // Read the input list of files and add them to the chain
1653 TString line;
1654 TChain *chain = new TChain(treeName);
1655 while (in.good())
1656 {
1657 in >> line;
1658 if (line.IsNull()) continue;
1659 if (count++ == fNtestFiles) break;
1660 TString esdFile(line);
1661 TFile *file = TFile::Open(esdFile);
1662 if (file) {
1663 if (!file->IsZombie()) chain->Add(esdFile);
1664 file->Close();
1665 } else {
1666 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1667 }
1668 }
1669 in.close();
1670 if (!chain->GetListOfFiles()->GetEntries()) {
1671 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1672 delete chain;
1673 return NULL;
1674 }
1675// chain->ls();
1676 return chain;
1677}
1678
d2a409b2 1679//______________________________________________________________________________
1680const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1681{
1682// Get job status for all jobs with jobid>jobidstart.
1683 static char mstatus[20];
1684 mstatus[0] = '\0';
1685 nrunning = 0;
1686 nwaiting = 0;
1687 nerror = 0;
1688 ndone = 0;
1689 TGridJobStatusList *list = gGrid->Ps("");
1690 if (!list) return mstatus;
1691 Int_t nentries = list->GetSize();
1692 TGridJobStatus *status;
1693 Int_t pid;
1694 for (Int_t ijob=0; ijob<nentries; ijob++) {
1695 status = (TGridJobStatus *)list->At(ijob);
8ddad121 1696 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
d2a409b2 1697 if (pid<jobidstart) continue;
1698 if (pid == lastid) {
8ddad121 1699 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
d2a409b2 1700 }
1701 switch (status->GetStatus()) {
1702 case TGridJobStatus::kWAITING:
1703 nwaiting++; break;
1704 case TGridJobStatus::kRUNNING:
1705 nrunning++; break;
1706 case TGridJobStatus::kABORTED:
1707 case TGridJobStatus::kFAIL:
1708 case TGridJobStatus::kUNKNOWN:
1709 nerror++; break;
1710 case TGridJobStatus::kDONE:
1711 ndone++;
1712 }
1713 }
1714 list->Delete();
1715 delete list;
1716 return mstatus;
1717}
1718
c57f56b7 1719//______________________________________________________________________________
1720Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1721{
1722// Returns true if file is a collection. Functionality duplicated from
1723// TAlien::Type() because we don't want to directly depend on TAlien.
1724 if (!gGrid) {
1725 Error("IsCollection", "No connection to grid");
1726 return kFALSE;
1727 }
1728 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1729 if (!res) return kFALSE;
1730 const char* typeStr = res->GetKey(0, "type");
1731 if (!typeStr || !strlen(typeStr)) return kFALSE;
1732 if (!strcmp(typeStr, "collection")) return kTRUE;
1733 delete res;
1734 return kFALSE;
1735}
1736
fe2d7fc2 1737//______________________________________________________________________________
1738Bool_t AliAnalysisAlien::IsSingleOutput() const
1739{
1740// Check if single-ouput option is on.
1741 return (!fOutputSingle.IsNull());
1742}
1743
16a4353c 1744//______________________________________________________________________________
1745void AliAnalysisAlien::Print(Option_t *) const
1746{
1747// Print current plugin settings.
84fcd93f 1748 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1750 if (mgr && mgr->IsProofMode()) {
1751 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1752 if (TestBit(AliAnalysisGrid::kTest))
1753 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1754 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1755 if (!fProofDataSet.IsNull())
1756 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1757 if (fProofReset==1)
1758 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1759 if (fProofReset>1)
1760 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (!fRootVersionForProof.IsNull())
1762 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1763 else
1764 printf("= ROOT version requested________________________ default\n");
1765 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1766 if (!fAliRootMode.IsNull())
1767 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1768 if (fNproofWorkers)
1769 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1770 if (fNproofWorkersPerSlave)
1771 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1772 if (TestSpecialBit(kClearPackages))
1773 printf("= ClearPackages requested...\n");
1774 if (fIncludePath.Data())
1775 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1776 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1777 if (fPackages && fPackages->GetEntries()) {
1778 TIter next(fPackages);
1779 TObject *obj;
1780 TString list;
1781 while ((obj=next())) list += obj->GetName();
1782 printf("= Par files to be used: ________________________ %s\n", list.Data());
1783 }
1784 if (TestSpecialBit(kProofConnectGrid))
1785 printf("= Requested PROOF connection to grid\n");
1786 return;
1787 }
e1c22e21 1788 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1789 if (fOverwriteMode) {
1790 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1791 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1792 }
348be253 1793 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1794 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1795 printf("= Production mode:______________________________ %d\n", fProductionMode);
1796 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1797 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1798 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1799 if (fUser.Length())
84fcd93f 1800 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1801 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1802 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1803 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1804 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1805 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1806 if (fRunNumbers.Length())
84fcd93f 1807 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1808 if (fRunRange[0])
74013535 1809 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1810 if (!fRunRange[0] && !fRunNumbers.Length()) {
1811 TIter next(fInputFiles);
1812 TObject *obj;
1813 TString list;
1814 while ((obj=next())) list += obj->GetName();
84fcd93f 1815 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1816 }
1817 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1818 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1819 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1820 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1821 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1822 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1823 printf("=====================================================================\n");
1824 printf("= Job price: ___________________________________ %d\n", fPrice);
1825 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1826 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1827 if (fMaxInitFailed>0)
84fcd93f 1828 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1829 if (fMasterResubmitThreshold>0)
84fcd93f 1830 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1831 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1832 if (fNrunsPerMaster>0)
84fcd93f 1833 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1834 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1835 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1836 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1837 if (fArguments.Length())
84fcd93f 1838 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1839 if (fExecutableArgs.Length())
1840 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1841 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1842 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1843 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1844 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1845 if (fDatasetName)
84fcd93f 1846 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1847 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1848 if (fIncludePath.Data())
84fcd93f 1849 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1850 if (fCloseSE.Length())
84fcd93f 1851 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1852 if (fFriendChainName.Length())
84fcd93f 1853 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1854 if (fPackages && fPackages->GetEntries()) {
16a4353c 1855 TIter next(fPackages);
1856 TObject *obj;
1857 TString list;
1858 while ((obj=next())) list += obj->GetName();
84fcd93f 1859 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1860 }
1861}
1862
c57f56b7 1863//______________________________________________________________________________
1864void AliAnalysisAlien::SetDefaults()
1865{
1866// Set default values for everything. What cannot be filled will be left empty.
1867 if (fGridJDL) delete fGridJDL;
1868 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1869 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1870 fPrice = 1;
1871 fTTL = 30000;
1872 fSplitMaxInputFileNumber = 100;
1873 fMaxInitFailed = 0;
1874 fMasterResubmitThreshold = 0;
bb885a9e 1875 fNtestFiles = 10;
149d288c 1876 fNreplicas = 2;
a8739e8a 1877 fRunRange[0] = 0;
1878 fRunRange[1] = 0;
8e3bd06a 1879 fRunPrefix = "%d";
319593fb 1880 fNrunsPerMaster = 1;
16a4353c 1881 fMaxMergeFiles = 100;
c57f56b7 1882 fRunNumbers = "";
1883 fExecutable = "analysis.sh";
0a1c1f7f 1884 fExecutableCommand = "root -b -q";
c57f56b7 1885 fArguments = "";
631c0b05 1886 fExecutableArgs = "";
c57f56b7 1887 fAnalysisMacro = "myAnalysis.C";
1888 fAnalysisSource = "";
1889 fAdditionalLibs = "";
1890 fSplitMode = "se";
1891 fAPIVersion = "";
1892 fROOTVersion = "";
1893 fAliROOTVersion = "";
1894 fUser = ""; // Your alien user name
1895 fGridWorkingDir = "";
1896 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1897 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1898 fFriendChainName = "";
c57f56b7 1899 fGridOutputDir = "output";
b83f84f0 1900 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1901 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1902 fInputFormat = "xml-single";
1903 fJDLName = "analysis.jdl";
c6cb3634 1904 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1905 fMergeExcludes = "";
0f389141 1906 fMergeViaJDL = 0;
348be253 1907 SetUseCopy(kTRUE);
1908 SetCheckCopy(kTRUE);
149d288c 1909 SetDefaultOutputs(kTRUE);
e1c22e21 1910 fOverwriteMode = 1;
c57f56b7 1911}
1912
0f389141 1913//______________________________________________________________________________
b3e07543 1914Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
0f389141 1915{
b3e07543 1916// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1917 // First check if the result is already in the output directory.
1918 if (FileExists(Form("%s/%s",aliendir,filename))) {
1919 printf("Final merged results found. Not merging again.\n");
7c2cd90a 1920 return kFALSE;
1921 }
b3e07543 1922 // Now check the last stage done.
1923 Int_t stage = 0;
1924 while (1) {
1925 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1926 stage++;
1927 }
1928 // Next stage of merging
1929 stage++;
1930 TString pattern = "*root_archive.zip";
1931 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1932 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1933 if (res) delete res;
1934 // Write standard output to file
1935 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1936 // Count the number of files inside
1937 ifstream ifile;
1938 ifile.open(Form("Stage_%d.xml",stage));
1939 if (!ifile.good()) {
1940 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
7c2cd90a 1941 return kFALSE;
b3e07543 1942 }
1943 TString line;
1944 Int_t nfiles = 0;
1945 while (!ifile.eof()) {
1946 ifile >> line;
1947 if (line.Contains("/event")) nfiles++;
7c2cd90a 1948 }
b3e07543 1949 ifile.close();
1950 if (!nfiles) {
1951 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1952 return kFALSE;
1953 } else {
1954 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
7ae54d70 1955 }
b3e07543 1956 // Copy the file in the output directory
1957 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1958 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1959 // Check if this is the last stage to be done.
1960 Bool_t laststage = (nfiles<nperchunk);
1961 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1962 if (laststage) {
1963 printf("### Submiting final merging stage %d\n", stage);
1964 TString finalJDL = jdl;
1965 finalJDL.ReplaceAll(".jdl", "_final.jdl");
03335324 1966 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
b3e07543 1967 Int_t jobId = SubmitSingleJob(query);
1968 if (!jobId) return kFALSE;
1969 } else {
1970 printf("### Submiting merging stage %d\n", stage);
03335324 1971 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
b3e07543 1972 Int_t jobId = SubmitSingleJob(query);
1973 if (!jobId) return kFALSE;
7c2cd90a 1974 }
b3e07543 1975 return kTRUE;
1976}
7c2cd90a 1977
1978//______________________________________________________________________________
1979Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1980{
1981// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1982 if (!gGrid) return 0;
1983 printf("=> %s ------> ",query);
1984 TGridResult *res = gGrid->Command(query);
1985 if (!res) return 0;
1986 TString jobId = res->GetKey(0,"jobId");
1987 delete res;
1988 if (jobId.IsNull()) {
1989 printf("submission failed. Reason:\n");
1990 gGrid->Stdout();
1991 gGrid->Stderr();
1992 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1993 return 0;
1994 }
1995 printf(" Job id: %s\n", jobId.Data());
1996 return atoi(jobId);
1997}
1998
1999//______________________________________________________________________________
b3e07543 2000Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
7c2cd90a 2001{
b3e07543 2002// Merge given output files from basedir. Basedir can be an alien output directory
2003// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2004// files in a group (ignored for xml input). Merging can be done in stages:
2005// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2006// stage=1 : works with an xml of all root_archive.zip in the output directory
2007// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
a2f5fc01 2008 TString outputFile = output;
0f389141 2009 TString command;
a2f5fc01 2010 TString outputChunk;
2011 TString previousChunk = "";
b3e07543 2012 TObjArray *listoffiles = new TObjArray();
2013// listoffiles->SetOwner();
a2f5fc01 2014 Int_t countChunk = 0;
2015 Int_t countZero = nmaxmerge;
0f389141 2016 Bool_t merged = kTRUE;
a2f5fc01 2017 Int_t index = outputFile.Index("@");
2018 if (index > 0) outputFile.Remove(index);
7c2cd90a 2019 TString inputFile = outputFile;
b3e07543 2020 TString sbasedir = basedir;
2021 if (sbasedir.Contains(".xml")) {
2022 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2023 nmaxmerge = 9999999;
2024 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2025 if (!coll) {
2026 ::Error("MergeOutput", "Input XML collection empty.");
2027 return kFALSE;
2028 }
2029 // Iterate grid collection
2030 while (coll->Next()) {
2031 TString fname = gSystem->DirName(coll->GetTURL());
2032 fname += "/";
2033 fname += inputFile;
2034 listoffiles->Add(new TNamed(fname.Data(),""));
2035 }
2036 } else {
2037 command = Form("find %s/ *%s", basedir, inputFile.Data());
2038 printf("command: %s\n", command.Data());
2039 TGridResult *res = gGrid->Command(command);
2040 if (!res) {
2041 ::Error("MergeOutput","No result for the find command\n");
2042 delete listoffiles;
2043 return kFALSE;
2044 }
2045 TIter nextmap(res);
2046 TMap *map = 0;
2047 while ((map=(TMap*)nextmap())) {
2048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2049 if (!objs || !objs->GetString().Length()) {
2050 // Nothing found - skip this output
2051 delete res;
2052 delete listoffiles;
2053 return kFALSE;
2054 }
2055 listoffiles->Add(new TNamed(objs->GetName(),""));
2056 }
2057 delete res;
2058 }
2059 if (!listoffiles->GetEntries()) {
7c2cd90a 2060 ::Error("MergeOutput","No result for the find command\n");
b3e07543 2061 delete listoffiles;
0f389141 2062 return kFALSE;
2063 }
2064
2065 TFileMerger *fm = 0;
b3e07543 2066 TIter next0(listoffiles);
2067 TObjArray *listoffilestmp = new TObjArray();
2068 listoffilestmp->SetOwner();
2069 TObject *nextfile;
2070 TString snextfile;
2071 // Keep only the files at upper level
2072 Int_t countChar = 0;
2073 while ((nextfile=next0())) {
2074 snextfile = nextfile->GetName();
2075 Int_t crtCount = snextfile.CountChar('/');
2076 if (nextfile == listoffiles->First()) countChar = crtCount;
2077 if (crtCount < countChar) countChar = crtCount;
2078 }
2079 next0.Reset();
2080 while ((nextfile=next0())) {
2081 snextfile = nextfile->GetName();
2082 Int_t crtCount = snextfile.CountChar('/');
2083 if (crtCount > countChar) {
2084 delete nextfile;
2085 continue;
2086 }
2087 listoffilestmp->Add(nextfile);
2088 }
2089 delete listoffiles;
2090 listoffiles = listoffilestmp; // Now contains 'good' files
2091 listoffiles->Print();
2092 TIter next(listoffiles);
7c2cd90a 2093 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 2094 outputChunk = outputFile;
2095 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 2096 // Check for existent temporary merge files
e1c22e21 2097 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 2098 // Preserve old merging functionality for stage 0.
2099 if (stage==0) {
2100 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2101 while (1) {
2102 // Skip as many input files as in a chunk
2ea55496 2103 for (Int_t counter=0; counter<nmaxmerge; counter++) {
b3e07543 2104 nextfile = next();
2105 if (!nextfile) {
2ea55496 2106 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
b3e07543 2107 delete listoffiles;
2ea55496 2108 return kFALSE;
2109 }
b3e07543 2110 snextfile = nextfile->GetName();
7c2cd90a 2111 }
2112 outputChunk = outputFile;
2113 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2114 countChunk++;
2115 if (gSystem->AccessPathName(outputChunk)) continue;
2116 // Merged file with chunks up to <countChunk> found
2117 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2118 previousChunk = outputChunk;
2119 break;
2120 }
2121 }
2122 countZero = nmaxmerge;
2123
b3e07543 2124 while ((nextfile=next())) {
2125 snextfile = nextfile->GetName();
2ea55496 2126 // Loop 'find' results and get next LFN
7c2cd90a 2127 if (countZero == nmaxmerge) {
2128 // First file in chunk - create file merger and add previous chunk if any.
2129 fm = new TFileMerger(kFALSE);
2130 fm->SetFastMethod(kTRUE);
2131 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2132 outputChunk = outputFile;
2133 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2134 }
2135 // If last file found, put merged results in the output file
b3e07543 2136 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
7c2cd90a 2137 // Add file to be merged and decrement chunk counter.
b3e07543 2138 fm->AddFile(snextfile);
7c2cd90a 2139 countZero--;
b3e07543 2140 if (countZero==0 || nextfile == listoffiles->Last()) {
7c2cd90a 2141 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2142 // Nothing found - skip this output
2143 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
b3e07543 2144 merged = kFALSE;
2145 break;
7c2cd90a 2146 }
2147 fm->OutputFile(outputChunk);
2148 // Merge the outputs, then go to next chunk
2149 if (!fm->Merge()) {
2150 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
b3e07543 2151 merged = kFALSE;
2152 break;
7c2cd90a 2153 } else {
2154 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2155 gSystem->Unlink(previousChunk);
2156 }
b3e07543 2157 if (nextfile == listoffiles->Last()) break;
7c2cd90a 2158 countChunk++;
2159 countZero = nmaxmerge;
2160 previousChunk = outputChunk;
0f389141 2161 }
0f389141 2162 }
b3e07543 2163 delete listoffiles;
2164 delete fm;
7c2cd90a 2165 return merged;
2166 }
2167 // Merging stage different than 0.
2168 // Move to the begining of the requested chunk.
7c2cd90a 2169 fm = new TFileMerger(kFALSE);
2170 fm->SetFastMethod(kTRUE);
b3e07543 2171 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2172 delete listoffiles;
7c2cd90a 2173 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2174 // Nothing found - skip this output
2175 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2176 delete fm;
2177 return kFALSE;
2178 }
b3e07543 2179 fm->OutputFile(outputFile);
7c2cd90a 2180 // Merge the outputs
2181 if (!fm->Merge()) {
2182 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2183 delete fm;
2184 return kFALSE;
2185 } else {
b3e07543 2186 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
0f389141 2187 }
7c2cd90a 2188 delete fm;
2189 return kTRUE;
0f389141 2190}
2191
c57f56b7 2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::MergeOutputs()
2194{
2195// Merge analysis outputs existing in the AliEn space.
2196 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2197 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2198 if (!Connect()) {
2199 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2200 return kFALSE;
0f389141 2201 }
c9e8f7fd 2202 if (fMergeViaJDL) {
2203 if (!TestBit(AliAnalysisGrid::kMerge)) {
2204 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2205 return kFALSE;
2206 }
2207 if (fProductionMode) {
2208 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2209 return kFALSE;
2210 }
0f389141 2211 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2212 if (!SubmitMerging()) return kFALSE;
0f389141 2213 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2214 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2215 return kFALSE;
c57f56b7 2216 }
2217 // Get the output path
d2a409b2 2218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2219 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2220 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2221 return kFALSE;
2222 }
2223 if (!fOutputFiles.Length()) {
2224 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2225 return kFALSE;
0f389141 2226 }
2227 // Check if fast read option was requested
e1c22e21 2228 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2229 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2230 if (fFastReadOption) {
d3339be3 2231 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2232 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
b3e07543 2233 gEnv->SetValue("XNet.ConnectTimeout",50);
2234 gEnv->SetValue("XNet.RequestTimeout",50);
0f389141 2235 gEnv->SetValue("XNet.MaxRedirectCount",2);
b3e07543 2236 gEnv->SetValue("XNet.ReconnectTimeout",50);
0f389141 2237 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2238 }
e8b839ab 2239 // Make sure we change the temporary directory
2240 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2241 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2242 TIter next(list);
2243 TObjString *str;
a2f5fc01 2244 TString outputFile;
c57f56b7 2245 Bool_t merged = kTRUE;
2246 while((str=(TObjString*)next())) {
a2f5fc01 2247 outputFile = str->GetString();
2248 Int_t index = outputFile.Index("@");
2249 if (index > 0) outputFile.Remove(index);
2250 TString outputChunk = outputFile;
2251 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2252 // Skip already merged outputs
a2f5fc01 2253 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2254 if (fOverwriteMode) {
a2f5fc01 2255 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2256 gSystem->Unlink(outputFile);
2257 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2258 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2259 outputChunk.Data());
2260 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2261 }
2262 } else {
a2f5fc01 2263 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2264 continue;
2265 }
2266 } else {
a2f5fc01 2267 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2268 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2269 outputChunk.Data());
2270 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2271 }
2272 }
bb885a9e 2273 if (fMergeExcludes.Length() &&
a2f5fc01 2274 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2275 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2276 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2277 if (!merged) {
2278 Error("MergeOutputs", "Terminate() will NOT be executed");
2279 return kFALSE;
ff07ec61 2280 }
2281 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2282 if (fileOpened) fileOpened->Close();
c57f56b7 2283 }
0f389141 2284 return kTRUE;
c57f56b7 2285}
2286
bb885a9e 2287//______________________________________________________________________________
2288void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2289{
2290// Use the output files connected to output containers from the analysis manager
2291// rather than the files defined by SetOutputFiles
2292 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2293 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2294 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2295}
2296
149d288c 2297//______________________________________________________________________________
2298void AliAnalysisAlien::SetOutputFiles(const char *list)
2299{
2300// Manually set the output files list.
2301// Removes duplicates. Not allowed if default outputs are not disabled.
2302 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2303 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2304 return;
2305 }
2306 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2307 fOutputFiles = "";
2308 TString slist = list;
2309 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2310 TObjArray *arr = slist.Tokenize(" ");
2311 TObjString *os;
2312 TIter next(arr);
2313 TString sout;
2314 while ((os=(TObjString*)next())) {
2315 sout = os->GetString();
2316 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2317 if (fOutputFiles.Contains(sout)) continue;
2318 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2319 fOutputFiles += sout;
2320 }
2321 delete arr;
f790bc1b 2322}
149d288c 2323
2324//______________________________________________________________________________
2325void AliAnalysisAlien::SetOutputArchive(const char *list)
2326{
2327// Manually set the output archive list. Free text - you are on your own...
2328// Not allowed if default outputs are not disabled.
2329 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2330 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2331 return;
2332 }
2333 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2334 fOutputArchive = list;
2335}
2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2339{
2340// Setting a prefered output SE is not allowed anymore.
2341 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2342}
2343
5513444a 2344//______________________________________________________________________________
2345Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2346{
2347// Start remote grid analysis.
3bdcb562 2348 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2349 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2350 if (!mgr || !mgr->IsInitialized()) {
2351 Error("StartAnalysis", "You need an initialized analysis manager for this");
2352 return kFALSE;
2353 }
2354 // Are we in PROOF mode ?
2355 if (mgr->IsProofMode()) {
90d50a8c 2356 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2357 if (fProofCluster.IsNull()) {
2358 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2359 return kFALSE;
2360 }
2361 if (fProofDataSet.IsNull() && !testMode) {
2362 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2363 return kFALSE;
2364 }
2365 // Set the needed environment
2366 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2367 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2368 if (fProofReset && !testMode) {
2369 if (fProofReset==1) {
2370 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2371 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2372 } else {
2373 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2374 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2375 }
2376 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2377 return kFALSE;
2378 }
2379 // Do we need to change the ROOT version ? The success of this cannot be checked.
2380 if (!fRootVersionForProof.IsNull() && !testMode) {
2381 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2382 fProofCluster.Data(), fRootVersionForProof.Data()));
2383 }
2384 // Connect to PROOF and check the status
2385 Long_t proof = 0;
27734f0e 2386 TString sworkers;
2387 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2388 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2389 if (!testMode) {
27734f0e 2390 if (!sworkers.IsNull())
2391 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2392 else
2393 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2394 } else {
2395 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2396 if (!proof) {
2397 Error("StartAnalysis", "Could not start PROOF in test mode");
2398 return kFALSE;
2399 }
2400 }
2401 if (!proof) {
2402 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2403 return kFALSE;
2404 }
27734f0e 2405 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2406 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2407 // Is dataset existing ?
2408 if (!testMode) {
2409 TString dataset = fProofDataSet;
2410 Int_t index = dataset.Index("#");
2411 if (index>=0) dataset.Remove(index);
830acc4c 2412// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2413// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2414// return kFALSE;
2415// }
2416// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2417 }
2418 // Is ClearPackages() needed ?
2419 if (TestSpecialBit(kClearPackages)) {
2420 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2421 gROOT->ProcessLine("gProof->ClearPackages();");
2422 }
2423 // Is a given aliroot mode requested ?
2424 TList optionsList;
1ac846e9 2425 TString parLibs;
830acc4c 2426 if (!fAliRootMode.IsNull()) {
3bdcb562 2427 TString alirootMode = fAliRootMode;
2428 if (alirootMode == "default") alirootMode = "";
2429 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2430 optionsList.SetOwner();
2431 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2432 // Check the additional libs to be loaded
2433 TString extraLibs;
1ac846e9 2434 Bool_t parMode = kFALSE;
3bdcb562 2435 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2436 // Parse the extra libs for .so
2437 if (fAdditionalLibs.Length()) {
2438 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2439 TIter next(list);
2440 TObjString *str;
1ac846e9 2441 while((str=(TObjString*)next())) {
2442 if (str->GetString().Contains(".so")) {
2443 if (parMode) {
2444 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2445 break;
2446 }
2447 TString stmp = str->GetName();
2448 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2449 stmp.ReplaceAll(".so","");
2450 if (!extraLibs.IsNull()) extraLibs += ":";
2451 extraLibs += stmp;
2452 continue;
2453 }
2454 if (str->GetString().Contains(".par")) {
2455 // The first par file found in the list will not allow any further .so
2456 parMode = kTRUE;
2457 if (!parLibs.IsNull()) parLibs += ":";
2458 parLibs += str->GetName();
2459 continue;
2460 }
3bdcb562 2461 }
2462 if (list) delete list;
2463 }
2464 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2465 // Check extra includes
2466 if (!fIncludePath.IsNull()) {
2467 TString includePath = fIncludePath;
2468 includePath.ReplaceAll(" ",":");
8ddad121 2469 includePath.ReplaceAll("$ALICE_ROOT","");
2470 includePath.ReplaceAll("${ALICE_ROOT}","");
2471 includePath.ReplaceAll("-I","");
830acc4c 2472 includePath.Strip(TString::kTrailing, ':');
2473 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2474 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2475 }
27734f0e 2476 // Check if connection to grid is requested
2477 if (TestSpecialBit(kProofConnectGrid))
2478 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2479 // Enable AliRoot par
2480 if (testMode) {
2481 // Enable proof lite package
2482 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2483 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2484 TNamed *obj = (TNamed*)optionsList.At(i);
2485 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2486 }
2487 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
8ddad121 2488 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
830acc4c 2489 Info("StartAnalysis", "AliRootProofLite enabled");
2490 } else {
2491 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2492 return kFALSE;
2493 }
2494 } else {
8ddad121 2495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2496 fAliROOTVersion.Data(), &optionsList))) {
830acc4c 2497 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2498 return kFALSE;
2499 }
27734f0e 2500 }
1ac846e9 2501 // Enable first par files from fAdditionalLibs
2502 if (!parLibs.IsNull()) {
2503 TObjArray *list = parLibs.Tokenize(":");
2504 TIter next(list);
2505 TObjString *package;
2506 while((package=(TObjString*)next())) {
40501bb7 2507 TString spkg = package->GetName();
2508 spkg.ReplaceAll(".par", "");
2509 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
b50a41c3 2510 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
4f136f18 2511 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2512 if (gROOT->ProcessLine(enablePackage)) {
1ac846e9 2513 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2514 return kFALSE;
2515 }
2516 } else {
2517 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2518 return kFALSE;
2519 }
2520 }
2521 if (list) delete list;
2522 }
3bdcb562 2523 } else {
2524 if (fAdditionalLibs.Contains(".so") && !testMode) {
2525 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2526 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2527 return kFALSE;
2528 }
2529 }
2530 // Enable par files if requested
2531 if (fPackages && fPackages->GetEntries()) {
2532 TIter next(fPackages);
2533 TObject *package;
2534 while ((package=next())) {
1ac846e9 2535 // Skip packages already enabled
2536 if (parLibs.Contains(package->GetName())) continue;
40501bb7 2537 TString spkg = package->GetName();
2538 spkg.ReplaceAll(".par", "");
2539 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3bdcb562 2540 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2541 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2542 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2543 return kFALSE;
2544 }
2545 } else {
2546 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2547 return kFALSE;
2548 }
2549 }
2550 }
2551 // Do we need to load analysis source files ?
2552 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2553 if (fAnalysisSource.Length()) {
2554 TObjArray *list = fAnalysisSource.Tokenize(" ");
2555 TIter next(list);
2556 TObjString *str;
2557 while((str=(TObjString*)next())) {
2558 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2559 }
2560 if (list) delete list;
2561 }
830acc4c 2562 if (testMode) {
2563 // Register dataset to proof lite.
2564 if (fFileForTestMode.IsNull()) {
2565 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2566 return kFALSE;
2567 }
2568 if (gSystem->AccessPathName(fFileForTestMode)) {
2569 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2570 return kFALSE;
2571 }
2572 TFileCollection *coll = new TFileCollection();
2573 coll->AddFromFile(fFileForTestMode);
8ddad121 2574 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
27734f0e 2575 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2576 }
3bdcb562 2577 return kTRUE;
2578 }
c57f56b7 2579
43da816a 2580 // Check if output files have to be taken from the analysis manager
2581 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
f790bc1b 2582 // Add output files and AOD files
2583 fOutputFiles = GetListOfFiles("outaod");
c07b9ce2 2584 // Add extra files registered to the analysis manager
f790bc1b 2585 TString extra = GetListOfFiles("ext");
2586 if (!extra.IsNull()) {
149d288c 2587 extra.ReplaceAll(".root", "*.root");
f790bc1b 2588 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
149d288c 2589 fOutputFiles += extra;
84fcd93f 2590 }
149d288c 2591 // Compose the output archive.
b83f84f0 2592 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2593 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2594 }
f7b1cbc2 2595// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2596 if (TestBit(AliAnalysisGrid::kOffline)) {
2597 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2598 \n there nor any job run. You can revise the JDL and analysis \
2599 \n macro then run the same in \"submit\" mode.");
2600 } else if (TestBit(AliAnalysisGrid::kTest)) {
2601 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2602 \n dataset.");
2603 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2604 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2605 \n space and job submitted.");
2606 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2607 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2608 if (fMergeViaJDL) CheckInputData();
5513444a 2609 return kTRUE;
c57f56b7 2610 } else {
2611 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2612 }
2613
348be253 2614 Print();
c57f56b7 2615 if (!Connect()) {
2616 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2617 return kFALSE;
16a4353c 2618 }
d3b18c4c 2619 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2620 if (!CheckInputData()) {
2621 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2622 return kFALSE;
c57f56b7 2623 }
d3339be3 2624 if (!CreateDataset(fDataPattern)) {
2625 TString serror;
2626 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2627 if (fRunNumbers.Length()) serror = "run numbers";
2628 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2629 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2630 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2631 return kFALSE;
2632 }
c57f56b7 2633 WriteAnalysisFile();
2634 WriteAnalysisMacro();
2635 WriteExecutable();
2636 WriteValidationScript();
0f389141 2637 if (fMergeViaJDL) {
2638 WriteMergingMacro();
2639 WriteMergeExecutable();
2640 WriteValidationScript(kTRUE);
2641 }
5513444a 2642 if (!CreateJDL()) return kFALSE;
2643 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2644 if (testMode) {
c57f56b7 2645 // Locally testing the analysis
2646 Info("StartAnalysis", "\n_______________________________________________________________________ \
2647 \n Running analysis script in a daughter shell as on a worker node \
2648 \n_______________________________________________________________________");
149d288c 2649 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2650 TIter next(list);
2651 TObjString *str;
a2f5fc01 2652 TString outputFile;
c57f56b7 2653 while((str=(TObjString*)next())) {
a2f5fc01 2654 outputFile = str->GetString();
2655 Int_t index = outputFile.Index("@");
2656 if (index > 0) outputFile.Remove(index);
2657 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2658 }
2659 delete list;
2660 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
f50e4bc8 2661 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
c57f56b7 2662// gSystem->Exec("cat stdout");
5513444a 2663 return kFALSE;
c57f56b7 2664 }
5513444a 2665 // Check if submitting is managed by LPM manager
a3e84053 2666 if (fProductionMode) {
5513444a 2667 TString prodfile = fJDLName;
2668 prodfile.ReplaceAll(".jdl", ".prod");
2669 WriteProductionFile(prodfile);
2670 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2671 return kFALSE;
2672 }
a8739e8a 2673 // Submit AliEn job(s)
d2a409b2 2674 gGrid->Cd(fGridOutputDir);
a8739e8a 2675 TGridResult *res;
c57f56b7 2676 TString jobID = "";
d2a409b2 2677 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2678 // Submit a given xml or a set of runs
a8739e8a 2679 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2680 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2681 if (res) {
2682 const char *cjobId = res->GetKey(0,"jobId");
2683 if (!cjobId) {
a03be957 2684 gGrid->Stdout();
2685 gGrid->Stderr();
a8739e8a 2686 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2687 return kFALSE;
a8739e8a 2688 } else {
2689 Info("StartAnalysis", "\n_______________________________________________________________________ \
2690 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2691 \n_______________________________________________________________________",
2692 fJDLName.Data(), cjobId);
2693 jobID = cjobId;
2694 }
2695 delete res;
a03be957 2696 } else {
2697 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2698 return kFALSE;
a8739e8a 2699 }
2700 } else {
d2a409b2 2701 // Submit for a range of enumeration of runs.
a03be957 2702 if (!Submit()) return kFALSE;
c57f56b7 2703 }
a8739e8a 2704
c57f56b7 2705 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2706 \n You may exit at any time and terminate the job later using the option <terminate> \
2707 \n ##################################################################################", jobID.Data());
bb885a9e 2708 gSystem->Exec("aliensh");
5513444a 2709 return kTRUE;
c57f56b7 2710}
2711
f790bc1b 2712//______________________________________________________________________________
2713const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2714{
2715// Get a comma-separated list of output files of the requested type.
2716// Type can be (case unsensitive):
2717// aod - list of aod files (std, extensions and filters)
2718// out - list of output files connected to containers (but not aod's or extras)
2719// ext - list of extra files registered to the manager
2720// ter - list of files produced in terminate
2721 static TString files;
2722 files = "";
2723 TString stype = type;
2724 stype.ToLower();
2725 TString aodfiles, extra;
2726 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2727 if (!mgr) {
2728 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2729 return files.Data();
2730 }
2731 if (mgr->GetOutputEventHandler()) {
2732 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2733 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2734 if (!extraaod.IsNull()) {
2735 aodfiles += ",";
2736 aodfiles += extraaod;
2737 }
2738 }
2739 if (stype.Contains("aod")) {
2740 files = aodfiles;
2741 if (stype == "aod") return files.Data();
2742 }
2743 // Add output files that are not in the list of AOD files
2744 TString outputfiles = "";
2745 TIter next(mgr->GetOutputs());
2746 AliAnalysisDataContainer *output;
2747 const char *filename = 0;
2748 while ((output=(AliAnalysisDataContainer*)next())) {
2749 filename = output->GetFileName();
2750 if (!(strcmp(filename, "default"))) continue;
2751 if (outputfiles.Contains(filename)) continue;
2752 if (aodfiles.Contains(filename)) continue;
2753 if (!outputfiles.IsNull()) outputfiles += ",";
2754 outputfiles += filename;
2755 }
2756 if (stype.Contains("out")) {
2757 if (!files.IsNull()) files += ",";
2758 files += outputfiles;
2759 if (stype == "out") return files.Data();
2760 }
2761 // Add extra files registered to the analysis manager
2762 TString sextra;
2763 extra = mgr->GetExtraFiles();
2764 if (!extra.IsNull()) {
2765 extra.Strip();
2766 extra.ReplaceAll(" ", ",");
2767 TObjArray *fextra = extra.Tokenize(",");
2768 TIter nextx(fextra);
2769 TObject *obj;
2770 while ((obj=nextx())) {
2771 if (aodfiles.Contains(obj->GetName())) continue;
2772 if (outputfiles.Contains(obj->GetName())) continue;
2773 if (sextra.Contains(obj->GetName())) continue;
2774 if (!sextra.IsNull()) sextra += ",";
2775 sextra += obj->GetName();
2776 }
2777 delete fextra;
2778 if (stype.Contains("ext")) {
2779 if (!files.IsNull()) files += ",";
2780 files += sextra;
2781 }
2782 }
2783 if (stype == "ext") return files.Data();
2784 TString termfiles;
2785 if (!fTerminateFiles.IsNull()) {
2786 fTerminateFiles.Strip();
2787 fTerminateFiles.ReplaceAll(" ",",");
9c939f2b 2788 TObjArray *fextra = fTerminateFiles.Tokenize(",");
f790bc1b 2789 TIter nextx(fextra);
2790 TObject *obj;
2791 while ((obj=nextx())) {
2792 if (aodfiles.Contains(obj->GetName())) continue;
2793 if (outputfiles.Contains(obj->GetName())) continue;
2794 if (termfiles.Contains(obj->GetName())) continue;
2795 if (sextra.Contains(obj->GetName())) continue;
2796 if (!termfiles.IsNull()) termfiles += ",";
2797 termfiles += obj->GetName();
2798 }
2799 delete fextra;
2800 }
2801 if (stype.Contains("ter")) {
2802 if (!files.IsNull() && !termfiles.IsNull()) {
2803 files += ",";
2804 files += termfiles;
2805 }
2806 }
2807 return files.Data();
2808}
2809
d2a409b2 2810//______________________________________________________________________________
a03be957 2811Bool_t AliAnalysisAlien::Submit()
d2a409b2 2812{
2813// Submit all master jobs.
2814 Int_t nmasterjobs = fInputFiles->GetEntries();
2815 Long_t tshoot = gSystem->Now();
a03be957 2816 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2817 while (fNsubmitted < nmasterjobs) {
2818 Long_t now = gSystem->Now();
2819 if ((now-tshoot)>30000) {
2820 tshoot = now;
a03be957 2821 if (!SubmitNext()) return kFALSE;
d2a409b2 2822 }
2823 }
a03be957 2824 return kTRUE;
d2a409b2 2825}
2826
0f389141 2827//______________________________________________________________________________
a03be957 2828Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2829{
2830// Submit all merging jobs.
2831 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2832 gGrid->Cd(fGridOutputDir);
2833 TString mergeJDLName = fExecutable;
2834 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
9f020952 2835 if (!fInputFiles) {
2836 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2837 return kFALSE;
2838 }
0f389141 2839 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2840 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2841 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2842 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2843 if (fOutputToRunNo) {
2844 // The output directory is the run number
2845 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2846 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2847 } else {
2848 // The output directory is the master number in 3 digits format
2849 printf("### Submitting merging job for master <%03d>\n", i);
2850 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2851 }
2852 // Check now the number of merging stages.
2b222f7c 2853 TObjArray *list = fOutputFiles.Tokenize(",");
2854 TIter next(list);
2855 TObjString *str;
2856 TString outputFile;
2857 while((str=(TObjString*)next())) {
2858 outputFile = str->GetString();
2859 Int_t index = outputFile.Index("@");
2860 if (index > 0) outputFile.Remove(index);
2861 if (!fMergeExcludes.Contains(outputFile)) break;
2862 }
2863 delete list;
b3e07543 2864 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
1d0b4d65 2865 if (!done && (i==ntosubmit-1)) return kFALSE;
0f389141 2866 }
a03be957 2867 if (!ntosubmit) return kTRUE;
9f020952 2868 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2869 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2870 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2871 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2872 \n ################################################################################################################");
0f389141 2873 gSystem->Exec("aliensh");
a03be957 2874 return kTRUE;
0f389141 2875}
2876
d2a409b2 2877//______________________________________________________________________________
a03be957 2878Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2879{
08d5b699 2880// Submit next bunch of master jobs if the queue is free. The first master job is
2881// submitted right away, while the next will not be unless the previous was split.
2882// The plugin will not submit new master jobs if there are more that 500 jobs in
2883// waiting phase.
d2a409b2 2884 static Bool_t iscalled = kFALSE;
2885 static Int_t firstmaster = 0;
2886 static Int_t lastmaster = 0;
2887 static Int_t npermaster = 0;
a03be957 2888 if (iscalled) return kTRUE;
d2a409b2 2889 iscalled = kTRUE;
2890 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2891 Int_t ntosubmit = 0;
2892 TGridResult *res;
2893 TString jobID = "";
3bdcb562 2894 Int_t nmasterjobs = fInputFiles->GetEntries();
2895 if (!fNsubmitted) {
2896 ntosubmit = 1;
2897 if (!IsUseSubmitPolicy()) {
90d50a8c 2898 if (nmasterjobs>5)
3bdcb562 2899 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2900 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2901 ntosubmit = nmasterjobs;
2902 }
2903 } else {
d2a409b2 2904 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2905 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2906 // If last master not split, just return
a03be957 2907 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2908 // No more than 100 waiting jobs
08d5b699 2909 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2910 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2911 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2912 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2913 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2914 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2915 }
d2a409b2 2916 for (Int_t i=0; i<ntosubmit; i++) {
2917 // Submit for a range of enumeration of runs.
a03be957 2918 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2919 TString query;
cd11251e 2920 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2921 runOutDir.ReplaceAll(".xml", "");
2922 if (fOutputToRunNo)
2923 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2924 else
2925 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2926 printf("********* %s\n",query.Data());
d2a409b2 2927 res = gGrid->Command(query);
2928 if (res) {
98ca124f 2929 TString cjobId1 = res->GetKey(0,"jobId");
2930 if (!cjobId1.Length()) {
d2a409b2 2931 iscalled = kFALSE;
a03be957 2932 gGrid->Stdout();
2933 gGrid->Stderr();
2934 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2935 return kFALSE;
d2a409b2 2936 } else {
2937 Info("StartAnalysis", "\n_______________________________________________________________________ \
2938 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2939 \n_______________________________________________________________________",
98ca124f 2940 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2941 jobID += cjobId1;
2942 jobID += " ";
98ca124f 2943 lastmaster = cjobId1.Atoi();
d2a409b2 2944 if (!firstmaster) firstmaster = lastmaster;
2945 fNsubmitted++;
2946 }
2947 delete res;
a03be957 2948 } else {
2949 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2950 return kFALSE;
d2a409b2 2951 }
2952 }
2953 iscalled = kFALSE;
a03be957 2954 return kTRUE;
d2a409b2 2955}
2956
c57f56b7 2957//______________________________________________________________________________
2958void AliAnalysisAlien::WriteAnalysisFile()
2959{
f10e8481 2960// Write current analysis manager into the file <analysisFile>
2961 TString analysisFile = fExecutable;
2962 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2963 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2964 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2965 if (!mgr || !mgr->IsInitialized()) {
2966 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2967 return;
2968 }
2969 // Check analysis type
2970 TObject *handler;
2971 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2972 handler = (TObject*)mgr->GetInputEventHandler();
2973 if (handler) {
2974 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2975 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2976 }
2977 TDirectory *cdir = gDirectory;
f10e8481 2978 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2979 if (file) {
e16a394c 2980 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2981 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2982 // Unless merging makes no sense
2983 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2984 mgr->Write();
2985 delete file;
fe2d7fc2 2986 // Enable termination for local jobs
2987 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2988 }
2989 if (cdir) cdir->cd();
f10e8481 2990 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2991 }
2992 Bool_t copy = kTRUE;
d3b18c4c 2993 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 2994 if (copy) {
2995 CdWork();
2996 TString workdir = gGrid->GetHomeDirectory();
2997 workdir += fGridWorkingDir;
d3b18c4c 2998 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 2999 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3000 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 3001 }
3002}
3003
3004//______________________________________________________________________________
3005void AliAnalysisAlien::WriteAnalysisMacro()
3006{
3007// Write the analysis macro that will steer the analysis in grid mode.
3008 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3009 ofstream out;
3010 out.open(fAnalysisMacro.Data(), ios::out);
3011 if (!out.good()) {
3012 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3013 return;
3014 }
5b273635 3015 Bool_t hasSTEERBase = kFALSE;
3016 Bool_t hasESD = kFALSE;
3017 Bool_t hasAOD = kFALSE;
3018 Bool_t hasANALYSIS = kFALSE;
3019 Bool_t hasANALYSISalice = kFALSE;
3020 Bool_t hasCORRFW = kFALSE;
c57f56b7 3021 TString func = fAnalysisMacro;
3022 TString type = "ESD";
3023 TString comment = "// Analysis using ";
c85cfc0f 3024 if (IsUseMCchain()) {
3025 type = "MC";
3026 comment += "MC";
3027 } else {
3028 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3029 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3030 type = "AOD";
3031 comment += "AOD";
3032 }
3033 }
0df6ccf2 3034 if (type!="AOD" && fFriendChainName!="") {
3035 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3036 return;
3037 }
c57f56b7 3038 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3039 else comment += " data";
3040 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3041 func.ReplaceAll(".C", "");
3042 out << "void " << func.Data() << "()" << endl;
3043 out << "{" << endl;
3044 out << comment.Data() << endl;
3045 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 3046 out << " TStopwatch timer;" << endl;
3047 out << " timer.Start();" << endl << endl;
13f0c00b 3048 // Change temp directory to current one
3049 out << "// Set temporary merging directory to current one" << endl;
3050 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
b3e07543 3051 // Reset existing include path
3052 out << "// Reset existing include path and add current directory first in the search" << endl;
3053 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
f790bc1b 3054 if (!fExecutableCommand.Contains("aliroot")) {
3055 out << "// load base root libraries" << endl;
3056 out << " gSystem->Load(\"libTree\");" << endl;
3057 out << " gSystem->Load(\"libGeom\");" << endl;
3058 out << " gSystem->Load(\"libVMC\");" << endl;
3059 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3060 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3061 }
d5c6455a 3062 if (fAdditionalRootLibs.Length()) {
47a4137d 3063 // in principle libtree /lib geom libvmc etc. can go into this list, too
3064 out << "// Add aditional libraries" << endl;
3065 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3066 TIter next(list);
3067 TObjString *str;
3068 while((str=(TObjString*)next())) {
3069 if (str->GetString().Contains(".so"))
3070 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 3071 }
47a4137d 3072 if (list) delete list;
d5c6455a 3073 }
57377eb5 3074 out << "// Load analysis framework libraries" << endl;
652561d5 3075 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 3076 if (!fPackages) {
f790bc1b 3077 if (!fExecutableCommand.Contains("aliroot")) {
3078 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3079 out << " gSystem->Load(\"libESD\");" << endl;
3080 out << " gSystem->Load(\"libAOD\");" << endl;
3081 }
4e5c5506 3082 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 3083 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3084 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 3085 } else {
4e5c5506 3086 TIter next(fPackages);
3087 TObject *obj;
57377eb5 3088 TString pkgname;
57377eb5 3089 while ((obj=next())) {
3090 pkgname = obj->GetName();
4478e6f1 3091 if (pkgname == "STEERBase" ||
3092 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3093 if (pkgname == "ESD" ||
3094 pkgname == "ESD.par") hasESD = kTRUE;
3095 if (pkgname == "AOD" ||
3096 pkgname == "AOD.par") hasAOD = kTRUE;
3097 if (pkgname == "ANALYSIS" ||
3098 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3099 if (pkgname == "ANALYSISalice" ||
3100 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3101 if (pkgname == "CORRFW" ||
3102 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 3103 }
3104 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 3105 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3106 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 3107 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3108 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 3109 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3110 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 3111 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3112 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 3113 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3114 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 3115 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3116 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 3117 out << "// Compile other par packages" << endl;
3118 next.Reset();
fcc9bb6f 3119 while ((obj=next())) {
3120 pkgname = obj->GetName();
4478e6f1 3121 if (pkgname == "STEERBase" ||
3122 pkgname == "STEERBase.par" ||
3123 pkgname == "ESD" ||
3124 pkgname == "ESD.par" ||
3125 pkgname == "AOD" ||
3126 pkgname == "AOD.par" ||
3127 pkgname == "ANALYSIS" ||
3128 pkgname == "ANALYSIS.par" ||
3129 pkgname == "ANALYSISalice" ||
3130 pkgname == "ANALYSISalice.par" ||
3131 pkgname == "CORRFW" ||
3132 pkgname == "CORRFW.par") continue;
5b273635 3133 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 3134 }
4e5c5506 3135 }
b3e07543 3136 out << "// include path" << endl;
3137 // Get the include path from the interpreter and remove entries pointing to AliRoot
3138 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3139 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3140 out << " TIter nextpath(listpaths);" << endl;
3141 out << " TObjString *pname;" << endl;
3142 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3143 out << " TString current = pname->GetName();" << endl;
3144 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3145 out << " gSystem->AddIncludePath(current);" << endl;
3146 out << " }" << endl;
3147 out << " if (listpaths) delete listpaths;" << endl;
3148 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3149 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3150 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
6da75e0b 3151 if (fAdditionalLibs.Length()) {
3152 out << "// Add aditional AliRoot libraries" << endl;
3153 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3154 TIter next(list);
3155 TObjString *str;
3156 while((str=(TObjString*)next())) {
3157 if (str->GetString().Contains(".so"))
3158 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 3159 if (str->GetString().Contains(".par"))
3160 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 3161 }
3162 if (list) delete list;
3163 }
3164 out << endl;
c57f56b7 3165 out << "// analysis source to be compiled at runtime (if any)" << endl;
3166 if (fAnalysisSource.Length()) {
3167 TObjArray *list = fAnalysisSource.Tokenize(" ");
3168 TIter next(list);
3169 TObjString *str;
3170 while((str=(TObjString*)next())) {
3171 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3172 }
3173 if (list) delete list;
3174 }
3175 out << endl;
b3e07543 3176// out << " printf(\"Currently load libraries:\\n\");" << endl;
3177// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
0f389141 3178 if (fFastReadOption) {
d3339be3 3179 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3180 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 3181 out << "// fast xrootd reading enabled" << endl;
3182 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3183 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3184 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3185 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3186 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3187 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3188 }
c57f56b7 3189 out << "// connect to AliEn and make the chain" << endl;
3190 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
c57f56b7 3191 out << "// read the analysis manager from file" << endl;
f10e8481 3192 TString analysisFile = fExecutable;
3193 analysisFile.ReplaceAll(".sh", ".root");
3194 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 3195 out << " if (!file) return;" << endl;
3196 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3197 out << " AliAnalysisManager *mgr = 0;" << endl;
3198 out << " TKey *key;" << endl;
3199 out << " while ((key=(TKey*)nextkey())) {" << endl;
3200 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3201 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3202 out << " };" << endl;
3203 out << " if (!mgr) {" << endl;
205b201f 3204 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 3205 out << " return;" << endl;
3206 out << " }" << endl << endl;
3207 out << " mgr->PrintStatus();" << endl;
52b6a92b 3208 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 3209 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 3210 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 3211 } else {
9a8288ff 3212 if (TestBit(AliAnalysisGrid::kTest))
3213 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3214 else
3215 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 3216 }
3217 }
242accb2 3218 if (IsUsingTags()) {
3219 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3220 } else {
3221 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3222 }
c57f56b7 3223 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 3224 out << " timer.Stop();" << endl;
3225 out << " timer.Print();" << endl;
c57f56b7 3226 out << "}" << endl << endl;
3227 if (IsUsingTags()) {
3228 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3229 out << "{" << endl;
3230 out << "// Create a chain using tags from the xml file." << endl;
3231 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3232 out << " if (!coll) {" << endl;
3233 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3234 out << " return NULL;" << endl;
3235 out << " }" << endl;
3236 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3237 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3238 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3239 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3240 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3241 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3242 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3243 out << " // Check if the cuts configuration file was provided" << endl;
3244 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3245 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3246 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3247 out << " }" << endl;
0df6ccf2 3248 if (fFriendChainName=="") {
3249 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3250 } else {
3251 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3252 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3253 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3254 }
c57f56b7 3255 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3256 out << " chain->ls();" << endl;
3257 out << " return chain;" << endl;
fcc9bb6f 3258 out << "}" << endl << endl;
c57f56b7 3259 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3260 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3261 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3262 msg += " AliLHCTagCuts *lhcCuts,\n";
3263 msg += " AliDetectorTagCuts *detCuts,\n";
3264 msg += " AliEventTagCuts *evCuts)";
23329835 3265 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 3266 }
0df6ccf2 3267 }
3268 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 3269 out <<"//________________________________________________________________________________" << endl;
c57f56b7 3270 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3271 out << "{" << endl;
3272 out << "// Create a chain using url's from xml file" << endl;
242accb2 3273 out << " TString filename;" << endl;
3274 out << " Int_t run = 0;" << endl;
c85cfc0f 3275 if (IsUseMCchain()) {
3276 out << " TString treename = \"TE\";" << endl;
3277 } else {
3278 out << " TString treename = type;" << endl;
3279 out << " treename.ToLower();" << endl;
3280 out << " treename += \"Tree\";" << endl;
3281 }
e02fee64 3282 out << " printf(\"***************************************\\n\");" << endl;
3283 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3284 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3285 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3286 out << " if (!coll) {" << endl;
3287 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3288 out << " return NULL;" << endl;
3289 out << " }" << endl;
242accb2 3290 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
c57f56b7 3291 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3292 if(fFriendChainName!="") {
3293 out << " TChain *chainFriend = new TChain(treename);" << endl;
3294 }
c57f56b7 3295 out << " coll->Reset();" << endl;
0df6ccf2 3296 out << " while (coll->Next()) {" << endl;
242accb2 3297 out << " filename = coll->GetTURL("");" << endl;
3298 out << " if (mgr) {" << endl;
3299 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3300 out << " if (nrun && nrun != run) {" << endl;
3301 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3302 out << " mgr->SetRunFromPath(nrun);" << endl;
3303 out << " run = nrun;" << endl;
3304 out << " }" << endl;
3305 out << " }" << endl;
3306 out << " chain->Add(filename);" << endl;
0df6ccf2 3307 if(fFriendChainName!="") {
3308 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3309 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3310 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3311 out << " chainFriend->Add(fileFriend.Data());" << endl;
3312 }
3313 out << " }" << endl;
c57f56b7 3314 out << " if (!chain->GetNtrees()) {" << endl;
3315 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3316 out << " return NULL;" << endl;
3317 out << " }" << endl;
0df6ccf2 3318 if(fFriendChainName!="") {
3319 out << " chain->AddFriend(chainFriend);" << endl;
3320 }
c57f56b7 3321 out << " return chain;" << endl;
fcc9bb6f 3322 out << "}" << endl << endl;
c57f56b7 3323 }
5b273635 3324 if (hasANALYSISalice) {
3325 out <<"//________________________________________________________________________________" << endl;
3326 out << "Bool_t SetupPar(const char *package) {" << endl;
3327 out << "// Compile the package and set it up." << endl;
3328 out << " TString pkgdir = package;" << endl;
3329 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3330 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3331 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3332 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3333 out << " // Check for BUILD.sh and execute" << endl;
3334 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3335 out << " printf(\"*******************************\\n\");" << endl;
3336 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3337 out << " printf(\"*******************************\\n\");" << endl;
3338 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3339 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3340 out << " gSystem->ChangeDirectory(cdir);" << endl;
3341 out << " return kFALSE;" << endl;
3342 out << " }" << endl;
3343 out << " } else {" << endl;
3344 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3345 out << " gSystem->ChangeDirectory(cdir);" << endl;
3346 out << " return kFALSE;" << endl;
3347 out << " }" << endl;
3348 out << " // Check for SETUP.C and execute" << endl;
3349 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3350 out << " printf(\"*******************************\\n\");" << endl;
3351 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3352 out << " printf(\"*******************************\\n\");" << endl;
3353 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3354 out << " } else {" << endl;
3355 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3356 out << " gSystem->ChangeDirectory(cdir);" << endl;
3357 out << " return kFALSE;" << endl;
3358 out << " }" << endl;
3359 out << " // Restore original workdir" << endl;
3360 out << " gSystem->ChangeDirectory(cdir);" << endl;
3361 out << " return kTRUE;" << endl;
3362 out << "}" << endl;
3363 }
c57f56b7 3364 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3365 }
3366 Bool_t copy = kTRUE;
d3b18c4c 3367 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3368 if (copy) {
3369 CdWork();
3370 TString workdir = gGrid->GetHomeDirectory();
3371 workdir += fGridWorkingDir;
3372 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3373 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3374 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3375 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3376 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3377 }
3378 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3379 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3380 }
3381}
3382
0f389141 3383//______________________________________________________________________________
3384void AliAnalysisAlien::WriteMergingMacro()
3385{
3386// Write a macro to merge the outputs per master job.
3387 if (!fMergeViaJDL) return;
3388 if (!fOutputFiles.Length()) {
3389 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3390 return;
3391 }
3392 TString mergingMacro = fExecutable;
3393 mergingMacro.ReplaceAll(".sh","_merge.C");
3394 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3395 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3396 ofstream out;
3397 out.open(mergingMacro.Data(), ios::out);
3398 if (!out.good()) {
3399 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3400 return;
3401 }
5b273635 3402 Bool_t hasSTEERBase = kFALSE;
3403 Bool_t hasESD = kFALSE;
3404 Bool_t hasAOD = kFALSE;
3405 Bool_t hasANALYSIS = kFALSE;
3406 Bool_t hasANALYSISalice = kFALSE;
3407 Bool_t hasCORRFW = kFALSE;
0f389141 3408 TString func = mergingMacro;
3409 TString comment;
3410 func.ReplaceAll(".C", "");
f47d5cb4 3411 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
0f389141 3412 out << "{" << endl;
3413 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3414 out << " TStopwatch timer;" << endl;
3415 out << " timer.Start();" << endl << endl;
b3e07543 3416 // Reset existing include path
3417 out << "// Reset existing include path and add current directory first in the search" << endl;
3418 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
7c2cd90a 3419 if (!fExecutableCommand.Contains("aliroot")) {
3420 out << "// load base root libraries" << endl;
3421 out << " gSystem->Load(\"libTree\");" << endl;
3422 out << " gSystem->Load(\"libGeom\");" << endl;
3423 out << " gSystem->Load(\"libVMC\");" << endl;
3424 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3425 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3426 }
0f389141 3427 if (fAdditionalRootLibs.Length()) {
3428 // in principle libtree /lib geom libvmc etc. can go into this list, too
3429 out << "// Add aditional libraries" << endl;
3430 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3431 TIter next(list);
3432 TObjString *str;
3433 while((str=(TObjString*)next())) {
3434 if (str->GetString().Contains(".so"))
3435 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3436 }
3437 if (list) delete list;
3438 }
0f389141 3439 out << "// Load analysis framework libraries" << endl;
3440 if (!fPackages) {
7c2cd90a 3441 if (!fExecutableCommand.Contains("aliroot")) {
3442 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3443 out << " gSystem->Load(\"libESD\");" << endl;
3444 out << " gSystem->Load(\"libAOD\");" << endl;
3445 }
0f389141 3446 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3447 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3448 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3449 } else {
3450 TIter next(fPackages);
3451 TObject *obj;
3452 TString pkgname;
5b273635 3453 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3454 while ((obj=next())) {
3455 pkgname = obj->GetName();
3456 if (pkgname == "STEERBase" ||
3457 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3458 if (pkgname == "ESD" ||
3459 pkgname == "ESD.par") hasESD = kTRUE;
3460 if (pkgname == "AOD" ||
3461 pkgname == "AOD.par") hasAOD = kTRUE;
3462 if (pkgname == "ANALYSIS" ||
3463 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3464 if (pkgname == "ANALYSISalice" ||
3465 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3466 if (pkgname == "CORRFW" ||
3467 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3468 }
5b273635 3469 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3470 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3471 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3472 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3473 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3474 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3475 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3476 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3477 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3478 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3479 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3480 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3481 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3482 out << "// Compile other par packages" << endl;
3483 next.Reset();
3484 while ((obj=next())) {
3485 pkgname = obj->GetName();
3486 if (pkgname == "STEERBase" ||
3487 pkgname == "STEERBase.par" ||
3488 pkgname == "ESD" ||
3489 pkgname == "ESD.par" ||
3490 pkgname == "AOD" ||
3491 pkgname == "AOD.par" ||
3492 pkgname == "ANALYSIS" ||
3493 pkgname == "ANALYSIS.par" ||
3494 pkgname == "ANALYSISalice" ||
3495 pkgname == "ANALYSISalice.par" ||
3496 pkgname == "CORRFW" ||
3497 pkgname == "CORRFW.par") continue;
5b273635 3498 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3499 }
3500 }
b3e07543 3501 out << "// include path" << endl;
3502 // Get the include path from the interpreter and remove entries pointing to AliRoot
3503 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3504 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3505 out << " TIter nextpath(listpaths);" << endl;
3506 out << " TObjString *pname;" << endl;
3507 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3508 out << " TString current = pname->GetName();" << endl;
3509 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3510 out << " gSystem->AddIncludePath(current);" << endl;
3511 out << " }" << endl;
3512 out << " if (listpaths) delete listpaths;" << endl;
3513 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3514 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3515 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
0f389141 3516 if (fAdditionalLibs.Length()) {
3517 out << "// Add aditional AliRoot libraries" << endl;
3518 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3519 TIter next(list);
3520 TObjString *str;
3521 while((str=(TObjString*)next())) {
3522 if (str->GetString().Contains(".so"))
3523 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3524 }
3525 if (list) delete list;
3526 }
3527 out << endl;
3528 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3529 if (fAnalysisSource.Length()) {
3530 TObjArray *list = fAnalysisSource.Tokenize(" ");
3531 TIter next(list);
3532 TObjString *str;
3533 while((str=(TObjString*)next())) {
3534 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3535 }
3536 if (list) delete list;
3537 }
149d288c 3538 out << endl;
3539
0f389141 3540 if (fFastReadOption) {
3541 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3542 out << "// fast xrootd reading enabled" << endl;
3543 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
b3e07543 3544 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3545 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
0f389141 3546 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
b3e07543 3547 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3548 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3549 }
3550 // Change temp directory to current one
3551 out << "// Set temporary merging directory to current one" << endl;
3552 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3553 out << "// Connect to AliEn" << endl;
3554 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
7c2cd90a 3555 out << " TString outputDir = dir;" << endl;
f790bc1b 3556 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
0f389141 3557 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
58268c13 3558 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3559 out << " TIter *iter = new TIter(list);" << endl;
3560 out << " TObjString *str;" << endl;
a2f5fc01 3561 out << " TString outputFile;" << endl;
0f389141 3562 out << " Bool_t merged = kTRUE;" << endl;
3563 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3564 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3565 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3566 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3567 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3568 out << " // Skip already merged outputs" << endl;
a2f5fc01 3569 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3570 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3571 out << " continue;" << endl;
3572 out << " }" << endl;
a2f5fc01 3573 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
b3e07543 3574 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
0f389141 3575 out << " if (!merged) {" << endl;
a2f5fc01 3576 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3577 out << " return;" << endl;
0f389141 3578 out << " }" << endl;
3579 out << " }" << endl;
7c2cd90a 3580 out << " // all outputs merged, validate" << endl;
3581 out << " ofstream out;" << endl;
3582 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3583 out << " out.close();" << endl;
3584 out << " // read the analysis manager from file" << endl;
f866cba5 3585 TString analysisFile = fExecutable;
3586 analysisFile.ReplaceAll(".sh", ".root");
f47d5cb4 3587 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
f866cba5 3588 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3589 out << " if (!file) return;" << endl;
f866cba5 3590 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3591 out << " AliAnalysisManager *mgr = 0;" << endl;
3592 out << " TKey *key;" << endl;
3593 out << " while ((key=(TKey*)nextkey())) {" << endl;
3594 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3595 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3596 out << " };" << endl;
3597 out << " if (!mgr) {" << endl;
3598 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3599 out << " return;" << endl;
3600 out << " }" << endl << endl;
5c4250fc 3601 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
b385fec0 3602 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3603 out << " mgr->PrintStatus();" << endl;
3604 if (AliAnalysisManager::GetAnalysisManager()) {
3605 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3606 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3607 } else {
9a8288ff 3608 if (TestBit(AliAnalysisGrid::kTest))
3609 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3610 else
3611 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3612 }
3613 }
9a8288ff 3614 out << " TTree *tree = NULL;" << endl;
3615 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3616 out << "}" << endl << endl;
5b273635 3617 if (hasANALYSISalice) {
3618 out <<"//________________________________________________________________________________" << endl;
3619 out << "Bool_t SetupPar(const char *package) {" << endl;
3620 out << "// Compile the package and set it up." << endl;
3621 out << " TString pkgdir = package;" << endl;
3622 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
b50a41c3 3623 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
5b273635 3624 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3625 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3626 out << " // Check for BUILD.sh and execute" << endl;
3627 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3628 out << " printf(\"*******************************\\n\");" << endl;
3629 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3630 out << " printf(\"*******************************\\n\");" << endl;
3631 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3632 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3633 out << " gSystem->ChangeDirectory(cdir);" << endl;
3634 out << " return kFALSE;" << endl;
3635 out << " }" << endl;
3636 out << " } else {" << endl;
3637 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3638 out << " gSystem->ChangeDirectory(cdir);" << endl;
3639 out << " return kFALSE;" << endl;
3640 out << " }" << endl;
3641 out << " // Check for SETUP.C and execute" << endl;
3642 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3643 out << " printf(\"*******************************\\n\");" << endl;
3644 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3645 out << " printf(\"*******************************\\n\");" << endl;
3646 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3647 out << " } else {" << endl;
3648 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3649 out << " gSystem->ChangeDirectory(cdir);" << endl;
3650 out << " return kFALSE;" << endl;
3651 out << " }" << endl;
3652 out << " // Restore original workdir" << endl;
3653 out << " gSystem->ChangeDirectory(cdir);" << endl;
3654 out << " return kTRUE;" << endl;
3655 out << "}" << endl;
3656 }
0f389141 3657 }
3658 Bool_t copy = kTRUE;
d3b18c4c 3659 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3660 if (copy) {
3661 CdWork();
3662 TString workdir = gGrid->GetHomeDirectory();
3663 workdir += fGridWorkingDir;
3664 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3665 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3666 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3667 }
3668}
3669
3670//______________________________________________________________________________
3671Bool_t AliAnalysisAlien::SetupPar(const char *package)
3672{
205b201f 3673// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3674// Note that for loading the compiled library. The current directory should have precedence in
3675// LD_LIBRARY_PATH
3676 TString pkgdir = package;
3677 pkgdir.ReplaceAll(".par","");
b50a41c3 3678 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
0f389141 3679 TString cdir = gSystem->WorkingDirectory();
3680 gSystem->ChangeDirectory(pkgdir);
3681 // Check for BUILD.sh and execute
3682 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3683 printf("**************************************************\n");
3684 printf("*** Building PAR archive %s\n", package);
3685 printf("**************************************************\n");
3686 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3687 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3688 gSystem->ChangeDirectory(cdir);
3689 return kFALSE;
3690 }
3691 } else {
3692 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3693 gSystem->ChangeDirectory(cdir);
3694 return kFALSE;
3695 }
3696 // Check for SETUP.C and execute
3697 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3698 printf("**************************************************\n");
3699 printf("*** Setup PAR archive %s\n", package);
3700 printf("**************************************************\n");
3701 gROOT->Macro("PROOF-INF/SETUP.C");
3702 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3703 } else {
3704 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3705 gSystem->ChangeDirectory(cdir);
3706 return kFALSE;
3707 }
3708 // Restore original workdir
3709 gSystem->ChangeDirectory(cdir);
3710 return kTRUE;
3711}
3712
c57f56b7 3713//______________________________________________________________________________
3714void AliAnalysisAlien::WriteExecutable()
3715{
3716// Generate the alien executable script.
3717 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3718 ofstream out;
3719 out.open(fExecutable.Data(), ios::out);
3720 if (out.bad()) {
5513444a 3721 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3722 return;
3723 }
3724 out << "#!/bin/bash" << endl;
2a4d5166 3725 // Make sure we can properly compile par files
3726 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
c57f56b7 3727 out << "echo \"=========================================\"" << endl;
3728 out << "echo \"############## PATH : ##############\"" << endl;
3729 out << "echo $PATH" << endl;
3730 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3731 out << "echo $LD_LIBRARY_PATH" << endl;
3732 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3733 out << "echo $ROOTSYS" << endl;
3734 out << "echo \"############## which root : ##############\"" << endl;
3735 out << "which root" << endl;
3736 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3737 out << "echo $ALICE_ROOT" << endl;
3738 out << "echo \"############## which aliroot : ##############\"" << endl;
3739 out << "which aliroot" << endl;
9c5ddadc 3740 out << "echo \"############## system limits : ##############\"" << endl;
3741 out << "ulimit -a" << endl;
3742 out << "echo \"############## memory : ##############\"" << endl;
3743 out << "free -m" << endl;
c57f56b7 3744 out << "echo \"=========================================\"" << endl << endl;
0a1c1f7f 3745 out << fExecutableCommand << " ";
631c0b05 3746 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3747 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3748 out << "echo \"############## memory after: ##############\"" << endl;
3749 out << "free -m" << endl;
c57f56b7 3750 }
3751 Bool_t copy = kTRUE;
d3b18c4c 3752 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3753 if (copy) {
3754 CdWork();
3755 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3756 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3757 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3758 workdir += fGridWorkingDir;
3759 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3760 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3761 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3762 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3763 }
3764}
3765
0f389141 3766//______________________________________________________________________________
3767void AliAnalysisAlien::WriteMergeExecutable()
3768{
3769// Generate the alien executable script for the merging job.
3770 if (!fMergeViaJDL) return;
3771 TString mergeExec = fExecutable;
3772 mergeExec.ReplaceAll(".sh", "_merge.sh");
3773 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3774 ofstream out;
3775 out.open(mergeExec.Data(), ios::out);
3776 if (out.bad()) {
3777 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3778 return;
3779 }
3780 out << "#!/bin/bash" << endl;
2a4d5166 3781 // Make sure we can properly compile par files
3782 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0f389141 3783 out << "echo \"=========================================\"" << endl;
3784 out << "echo \"############## PATH : ##############\"" << endl;
3785 out << "echo $PATH" << endl;
3786 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3787 out << "echo $LD_LIBRARY_PATH" << endl;
3788 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3789 out << "echo $ROOTSYS" << endl;
3790 out << "echo \"############## which root : ##############\"" << endl;
3791 out << "which root" << endl;
3792 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3793 out << "echo $ALICE_ROOT" << endl;
3794 out << "echo \"############## which aliroot : ##############\"" << endl;
3795 out << "which aliroot" << endl;
3796 out << "echo \"############## system limits : ##############\"" << endl;
3797 out << "ulimit -a" << endl;
3798 out << "echo \"############## memory : ##############\"" << endl;
3799 out << "free -m" << endl;
3800 out << "echo \"=========================================\"" << endl << endl;
0f389141 3801 TString mergeMacro = fExecutable;
3802 mergeMacro.ReplaceAll(".sh", "_merge.C");
f790bc1b 3803 if (IsOneStageMerging())
3804 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3805 else
f47d5cb4 3806 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
0f389141 3807 out << fExecutableCommand << " " << "$ARG" << endl;
3808 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3809 out << "echo \"############## memory after: ##############\"" << endl;
3810 out << "free -m" << endl;
0f389141 3811 }
3812 Bool_t copy = kTRUE;
d3b18c4c 3813 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3814 if (copy) {
3815 CdWork();
3816 TString workdir = gGrid->GetHomeDirectory();
3817 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3818 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3819 workdir += fGridWorkingDir;
3820 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3821 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3822 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3823 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3824 }
3825}
3826
c57f56b7 3827//______________________________________________________________________________
5513444a 3828void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3829{
3830// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3831// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3832// Next lines: full_path_to_dataset XXX (XXX is a string)
3833// To submit, one has to: submit jdl XXX for all lines
3834 ofstream out;
3835 out.open(filename, ios::out);
3836 if (out.bad()) {
3837 Error("WriteProductionFile", "Bad file name: %s", filename);
3838 return;
3839 }
d3b18c4c 3840 TString workdir;
3841 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3842 workdir = gGrid->GetHomeDirectory();
5513444a 3843 workdir += fGridWorkingDir;
f5e8c702 3844 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3845 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3846 out << locjdl << " " << njobspermaster << endl;
5513444a 3847 Int_t nmasterjobs = fInputFiles->GetEntries();
3848 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3849 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3850 runOutDir.ReplaceAll(".xml", "");
3851 if (fOutputToRunNo)
3852 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3853 else
3854 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3855 }
d3b18c4c 3856 if (gGrid) {
3857 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3858 if (FileExists(filename)) gGrid->Rm(filename);
3859 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3860 }
5513444a 3861}
3862
3863//______________________________________________________________________________
0f389141 3864void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3865{
3866// Generate the alien validation script.
3867 // Generate the validation script
3868 TObjString *os;
d3b18c4c 3869 if (fValidationScript.IsNull()) {
3870 fValidationScript = fExecutable;
3871 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3872 }
3873 TString validationScript = fValidationScript;
3874 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3875 if (!Connect()) {
3876 Error("WriteValidationScript", "Alien connection required");
3877 return;
3878 }
d3b18c4c 3879 if (!fTerminateFiles.IsNull()) {
3880 fTerminateFiles.Strip();
3881 fTerminateFiles.ReplaceAll(" ",",");
3882 }
a2f5fc01 3883 TString outStream = "";
3884 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3885 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3886 ofstream out;
0d5d317c 3887 out.open(validationScript, ios::out);
c57f56b7 3888 out << "#!/bin/bash" << endl;
3889 out << "##################################################" << endl;
3890 out << "validateout=`dirname $0`" << endl;
3891 out << "validatetime=`date`" << endl;
3892 out << "validated=\"0\";" << endl;
3893 out << "error=0" << endl;
3894 out << "if [ -z $validateout ]" << endl;
3895 out << "then" << endl;
3896 out << " validateout=\".\"" << endl;
3897 out << "fi" << endl << endl;
3898 out << "cd $validateout;" << endl;
3899 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3900 out << "echo \"*******************************************************\"" << outStream << endl;
3901 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3902 out << "" << endl;
a2f5fc01 3903 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3904 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3905 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3906 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3907 out << "ls -la ./" << outStream << endl;
3908 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3909 out << "##################################################" << endl;
ebec370a 3910 out << "" << endl;
3911
3912 out << "if [ ! -f stderr ] ; then" << endl;
3913 out << " error=1" << endl;
a2f5fc01 3914 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3915 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3916 out << "fi" << endl;
3917
b34c9f51 3918 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3919 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3920 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3921 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3922 out << "" << endl;
3923
ebec370a 3924 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3925 out << " error=1" << endl;
a2f5fc01 3926 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3927 out << " echo \"$parArch\" " << outStream << endl;
3928 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3929 out << "fi" << endl;
3930
3931 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3932 out << " error=1" << endl;
a2f5fc01 3933 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3934 out << " echo \"$segViol\" " << outStream << endl;
3935 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3936 out << "fi" << endl;
3937
3938 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3939 out << " error=1" << endl;
a2f5fc01 3940 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3941 out << " echo \"$segFault\" " << outStream << endl;
3942 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3943 out << "fi" << endl;
3944
b34c9f51 3945 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3946 out << " error=1" << endl;
a2f5fc01 3947 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3948 out << " echo \"$glibcErr\" " << outStream << endl;
3949 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3950 out << "fi" << endl;
3951
ebec370a 3952 // Part dedicated to the specific analyses running into the train
3953
d3b18c4c 3954 TString outputFiles = fOutputFiles;
3955 if (merge && !fTerminateFiles.IsNull()) {
3956 outputFiles += ",";
3957 outputFiles += fTerminateFiles;
3958 }
3959 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3960 TIter next1(arr);
a2f5fc01 3961 TString outputFile;
bbd88db8 3962 while (!merge && (os=(TObjString*)next1())) {
3963 // No need to validate outputs produced by merging since the merging macro does this
a2f5fc01 3964 outputFile = os->GetString();
3965 Int_t index = outputFile.Index("@");
3966 if (index > 0) outputFile.Remove(index);
bbd88db8 3967 if (fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3968 if (outputFile.Contains("*")) continue;
3969 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3970 out << " error=1" << endl;
7c2cd90a 3971 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3972 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3973 out << "fi" << endl;
3974 }
3975 delete arr;
7c2cd90a 3976 out << "if ! [ -f outputs_valid ] ; then" << endl;
3977 out << " error=1" << endl;
3978 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3979 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3980 out << "fi" << endl;
923e2ca5 3981
c57f56b7 3982 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3983 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3984 if (!IsKeepLogs()) {
3985 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3986 outStream = "";
149d288c 3987 out << " rm -f std*" << endl;
3988 }
c57f56b7 3989 out << "fi" << endl;
3990
a2f5fc01 3991 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3992 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3993 out << "cd -" << endl;
3994 out << "exit $error" << endl;
3995 }
3996 Bool_t copy = kTRUE;
d3b18c4c 3997 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3998 if (copy) {
3999 CdWork();
4000 TString workdir = gGrid->GetHomeDirectory();
4001 workdir += fGridWorkingDir;
d3b18c4c 4002 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 4003 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4004 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 4005 }
4006}