]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
New working version of CDBconnect task. Has to be run in sync with the change in...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
d2a409b2 61 fNsubmitted(0),
a3e84053 62 fProductionMode(0),
cd11251e 63 fOutputToRunNo(0),
0f389141 64 fMergeViaJDL(0),
65 fFastReadOption(0),
e1c22e21 66 fOverwriteMode(1),
149d288c 67 fNreplicas(2),
3bdcb562 68 fNproofWorkers(0),
27734f0e 69 fNproofWorkersPerSlave(0),
3bdcb562 70 fProofReset(0),
c57f56b7 71 fRunNumbers(),
72 fExecutable(),
0a1c1f7f 73 fExecutableCommand(),
c57f56b7 74 fArguments(),
631c0b05 75 fExecutableArgs(),
c57f56b7 76 fAnalysisMacro(),
77 fAnalysisSource(),
d3b18c4c 78 fValidationScript(),
d5c6455a 79 fAdditionalRootLibs(),
c57f56b7 80 fAdditionalLibs(),
81 fSplitMode(),
82 fAPIVersion(),
83 fROOTVersion(),
84 fAliROOTVersion(),
648174cf 85 fExternalPackages(),
c57f56b7 86 fUser(),
87 fGridWorkingDir(),
88 fGridDataDir(),
89 fDataPattern(),
90 fGridOutputDir(),
91 fOutputArchive(),
92 fOutputFiles(),
93 fInputFormat(),
e7c71df0 94 fDatasetName(),
c57f56b7 95 fJDLName(),
d3b18c4c 96 fTerminateFiles(),
bb885a9e 97 fMergeExcludes(),
f965131e 98 fIncludePath(),
bb885a9e 99 fCloseSE(),
0df6ccf2 100 fFriendChainName(),
c6cb3634 101 fJobTag(),
648174cf 102 fOutputSingle(),
5fce53f4 103 fRunPrefix(),
3bdcb562 104 fProofCluster(),
105 fProofDataSet(),
106 fFileForTestMode(),
107 fRootVersionForProof(),
108 fAliRootMode(),
4e5c5506 109 fInputFiles(0),
110 fPackages(0)
c57f56b7 111{
112// Dummy ctor.
113 SetDefaults();
114}
115
116//______________________________________________________________________________
117AliAnalysisAlien::AliAnalysisAlien(const char *name)
118 :AliAnalysisGrid(name),
119 fGridJDL(NULL),
0f389141 120 fMergingJDL(NULL),
c57f56b7 121 fPrice(0),
122 fTTL(0),
123 fSplitMaxInputFileNumber(0),
124 fMaxInitFailed(0),
125 fMasterResubmitThreshold(0),
bb885a9e 126 fNtestFiles(0),
319593fb 127 fNrunsPerMaster(0),
16a4353c 128 fMaxMergeFiles(0),
d2a409b2 129 fNsubmitted(0),
a3e84053 130 fProductionMode(0),
cd11251e 131 fOutputToRunNo(0),
0f389141 132 fMergeViaJDL(0),
133 fFastReadOption(0),
e1c22e21 134 fOverwriteMode(1),
149d288c 135 fNreplicas(2),
3bdcb562 136 fNproofWorkers(0),
27734f0e 137 fNproofWorkersPerSlave(0),
3bdcb562 138 fProofReset(0),
c57f56b7 139 fRunNumbers(),
140 fExecutable(),
0a1c1f7f 141 fExecutableCommand(),
c57f56b7 142 fArguments(),
631c0b05 143 fExecutableArgs(),
c57f56b7 144 fAnalysisMacro(),
145 fAnalysisSource(),
d3b18c4c 146 fValidationScript(),
d5c6455a 147 fAdditionalRootLibs(),
c57f56b7 148 fAdditionalLibs(),
149 fSplitMode(),
150 fAPIVersion(),
151 fROOTVersion(),
152 fAliROOTVersion(),
648174cf 153 fExternalPackages(),
c57f56b7 154 fUser(),
155 fGridWorkingDir(),
156 fGridDataDir(),
157 fDataPattern(),
158 fGridOutputDir(),
159 fOutputArchive(),
160 fOutputFiles(),
161 fInputFormat(),
e7c71df0 162 fDatasetName(),
c57f56b7 163 fJDLName(),
d3b18c4c 164 fTerminateFiles(),
bb885a9e 165 fMergeExcludes(),
f965131e 166 fIncludePath(),
bb885a9e 167 fCloseSE(),
0df6ccf2 168 fFriendChainName(),
c6cb3634 169 fJobTag(),
648174cf 170 fOutputSingle(),
5fce53f4 171 fRunPrefix(),
3bdcb562 172 fProofCluster(),
173 fProofDataSet(),
174 fFileForTestMode(),
175 fRootVersionForProof(),
176 fAliRootMode(),
4e5c5506 177 fInputFiles(0),
178 fPackages(0)
c57f56b7 179{
180// Default ctor.
181 SetDefaults();
182}
183
184//______________________________________________________________________________
185AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
186 :AliAnalysisGrid(other),
187 fGridJDL(NULL),
0f389141 188 fMergingJDL(NULL),
c57f56b7 189 fPrice(other.fPrice),
190 fTTL(other.fTTL),
191 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
192 fMaxInitFailed(other.fMaxInitFailed),
193 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 194 fNtestFiles(other.fNtestFiles),
319593fb 195 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 196 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 197 fNsubmitted(other.fNsubmitted),
a3e84053 198 fProductionMode(other.fProductionMode),
cd11251e 199 fOutputToRunNo(other.fOutputToRunNo),
0f389141 200 fMergeViaJDL(other.fMergeViaJDL),
201 fFastReadOption(other.fFastReadOption),
202 fOverwriteMode(other.fOverwriteMode),
149d288c 203 fNreplicas(other.fNreplicas),
3bdcb562 204 fNproofWorkers(other.fNproofWorkers),
27734f0e 205 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 206 fProofReset(other.fProofReset),
c57f56b7 207 fRunNumbers(other.fRunNumbers),
208 fExecutable(other.fExecutable),
0a1c1f7f 209 fExecutableCommand(other.fExecutableCommand),
c57f56b7 210 fArguments(other.fArguments),
631c0b05 211 fExecutableArgs(other.fExecutableArgs),
c57f56b7 212 fAnalysisMacro(other.fAnalysisMacro),
213 fAnalysisSource(other.fAnalysisSource),
d3b18c4c 214 fValidationScript(other.fValidationScript),
d5c6455a 215 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 216 fAdditionalLibs(other.fAdditionalLibs),
217 fSplitMode(other.fSplitMode),
218 fAPIVersion(other.fAPIVersion),
219 fROOTVersion(other.fROOTVersion),
220 fAliROOTVersion(other.fAliROOTVersion),
648174cf 221 fExternalPackages(other.fExternalPackages),
c57f56b7 222 fUser(other.fUser),
223 fGridWorkingDir(other.fGridWorkingDir),
224 fGridDataDir(other.fGridDataDir),
225 fDataPattern(other.fDataPattern),
226 fGridOutputDir(other.fGridOutputDir),
227 fOutputArchive(other.fOutputArchive),
228 fOutputFiles(other.fOutputFiles),
229 fInputFormat(other.fInputFormat),
e7c71df0 230 fDatasetName(other.fDatasetName),
c57f56b7 231 fJDLName(other.fJDLName),
d3b18c4c 232 fTerminateFiles(other.fTerminateFiles),
bb885a9e 233 fMergeExcludes(other.fMergeExcludes),
f965131e 234 fIncludePath(other.fIncludePath),
bb885a9e 235 fCloseSE(other.fCloseSE),
0df6ccf2 236 fFriendChainName(other.fFriendChainName),
c6cb3634 237 fJobTag(other.fJobTag),
648174cf 238 fOutputSingle(other.fOutputSingle),
5fce53f4 239 fRunPrefix(other.fRunPrefix),
3bdcb562 240 fProofCluster(other.fProofCluster),
241 fProofDataSet(other.fProofDataSet),
242 fFileForTestMode(other.fFileForTestMode),
243 fRootVersionForProof(other.fRootVersionForProof),
244 fAliRootMode(other.fAliRootMode),
4e5c5506 245 fInputFiles(0),
246 fPackages(0)
c57f56b7 247{
248// Copy ctor.
249 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 250 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 251 fRunRange[0] = other.fRunRange[0];
252 fRunRange[1] = other.fRunRange[1];
c57f56b7 253 if (other.fInputFiles) {
254 fInputFiles = new TObjArray();
255 TIter next(other.fInputFiles);
256 TObject *obj;
257 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
258 fInputFiles->SetOwner();
259 }
4e5c5506 260 if (other.fPackages) {
261 fPackages = new TObjArray();
262 TIter next(other.fPackages);
263 TObject *obj;
264 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
265 fPackages->SetOwner();
266 }
c57f56b7 267}
268
269//______________________________________________________________________________
270AliAnalysisAlien::~AliAnalysisAlien()
271{
272// Destructor.
273 if (fGridJDL) delete fGridJDL;
0f389141 274 if (fMergingJDL) delete fMergingJDL;
c57f56b7 275 if (fInputFiles) delete fInputFiles;
4e5c5506 276 if (fPackages) delete fPackages;
c57f56b7 277}
278
279//______________________________________________________________________________
280AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
281{
282// Assignment.
283 if (this != &other) {
284 AliAnalysisGrid::operator=(other);
285 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 286 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 287 fPrice = other.fPrice;
288 fTTL = other.fTTL;
289 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
290 fMaxInitFailed = other.fMaxInitFailed;
291 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 292 fNtestFiles = other.fNtestFiles;
a3e84053 293 fNrunsPerMaster = other.fNrunsPerMaster;
294 fMaxMergeFiles = other.fMaxMergeFiles;
295 fNsubmitted = other.fNsubmitted;
296 fProductionMode = other.fProductionMode;
cd11251e 297 fOutputToRunNo = other.fOutputToRunNo;
0f389141 298 fMergeViaJDL = other.fMergeViaJDL;
299 fFastReadOption = other.fFastReadOption;
300 fOverwriteMode = other.fOverwriteMode;
149d288c 301 fNreplicas = other.fNreplicas;
3bdcb562 302 fNproofWorkers = other.fNproofWorkers;
27734f0e 303 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 304 fProofReset = other.fProofReset;
c57f56b7 305 fRunNumbers = other.fRunNumbers;
306 fExecutable = other.fExecutable;
0a1c1f7f 307 fExecutableCommand = other.fExecutableCommand;
c57f56b7 308 fArguments = other.fArguments;
631c0b05 309 fExecutableArgs = other.fExecutableArgs;
c57f56b7 310 fAnalysisMacro = other.fAnalysisMacro;
311 fAnalysisSource = other.fAnalysisSource;
d3b18c4c 312 fValidationScript = other.fValidationScript;
d5c6455a 313 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 314 fAdditionalLibs = other.fAdditionalLibs;
315 fSplitMode = other.fSplitMode;
316 fAPIVersion = other.fAPIVersion;
317 fROOTVersion = other.fROOTVersion;
318 fAliROOTVersion = other.fAliROOTVersion;
648174cf 319 fExternalPackages = other.fExternalPackages;
c57f56b7 320 fUser = other.fUser;
321 fGridWorkingDir = other.fGridWorkingDir;
322 fGridDataDir = other.fGridDataDir;
323 fDataPattern = other.fDataPattern;
324 fGridOutputDir = other.fGridOutputDir;
325 fOutputArchive = other.fOutputArchive;
326 fOutputFiles = other.fOutputFiles;
327 fInputFormat = other.fInputFormat;
e7c71df0 328 fDatasetName = other.fDatasetName;
c57f56b7 329 fJDLName = other.fJDLName;
d3b18c4c 330 fTerminateFiles = other.fTerminateFiles;
bb885a9e 331 fMergeExcludes = other.fMergeExcludes;
f965131e 332 fIncludePath = other.fIncludePath;
bb885a9e 333 fCloseSE = other.fCloseSE;
0df6ccf2 334 fFriendChainName = other.fFriendChainName;
c6cb3634 335 fJobTag = other.fJobTag;
648174cf 336 fOutputSingle = other.fOutputSingle;
5fce53f4 337 fRunPrefix = other.fRunPrefix;
3bdcb562 338 fProofCluster = other.fProofCluster;
339 fProofDataSet = other.fProofDataSet;
340 fFileForTestMode = other.fFileForTestMode;
341 fRootVersionForProof = other.fRootVersionForProof;
342 fAliRootMode = other.fAliRootMode;
c57f56b7 343 if (other.fInputFiles) {
344 fInputFiles = new TObjArray();
345 TIter next(other.fInputFiles);
346 TObject *obj;
347 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
348 fInputFiles->SetOwner();
349 }
4e5c5506 350 if (other.fPackages) {
351 fPackages = new TObjArray();
352 TIter next(other.fPackages);
353 TObject *obj;
354 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
355 fPackages->SetOwner();
356 }
c57f56b7 357 }
358 return *this;
359}
360
f965131e 361//______________________________________________________________________________
362void AliAnalysisAlien::AddIncludePath(const char *path)
363{
364// Add include path in the remote analysis macro.
365 TString p(path);
366 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
367 else fIncludePath += Form("-I%s ", path);
368}
369
c57f56b7 370//______________________________________________________________________________
371void AliAnalysisAlien::AddRunNumber(Int_t run)
372{
373// Add a run number to the list of runs to be processed.
374 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 375 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 376}
377
ee75cfc3 378//______________________________________________________________________________
379void AliAnalysisAlien::AddRunNumber(const char* run)
380{
381// Add a run number to the list of runs to be processed.
382 if (fRunNumbers.Length()) fRunNumbers += " ";
383 fRunNumbers += run;
384}
385
c57f56b7 386//______________________________________________________________________________
387void AliAnalysisAlien::AddDataFile(const char *lfn)
388{
389// Adds a data file to the input to be analysed. The file should be a valid LFN
390// or point to an existing file in the alien workdir.
391 if (!fInputFiles) fInputFiles = new TObjArray();
392 fInputFiles->Add(new TObjString(lfn));
393}
648174cf 394
395//______________________________________________________________________________
396void AliAnalysisAlien::AddExternalPackage(const char *package)
397{
398// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
399 if (fExternalPackages) fExternalPackages += " ";
400 fExternalPackages += package;
401}
402
c57f56b7 403//______________________________________________________________________________
404Bool_t AliAnalysisAlien::Connect()
405{
406// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
407 if (gGrid && gGrid->IsConnected()) return kTRUE;
d3b18c4c 408 if (fProductionMode) return kTRUE;
c57f56b7 409 if (!gGrid) {
410 Info("Connect", "Trying to connect to AliEn ...");
411 TGrid::Connect("alien://");
412 }
413 if (!gGrid || !gGrid->IsConnected()) {
414 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
415 return kFALSE;
416 }
417 fUser = gGrid->GetUser();
418 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
419 return kTRUE;
420}
421
422//______________________________________________________________________________
423void AliAnalysisAlien::CdWork()
424{
425// Check validity of alien workspace. Create directory if possible.
426 if (!Connect()) {
427 Error("CdWork", "Alien connection required");
428 return;
429 }
430 TString homedir = gGrid->GetHomeDirectory();
431 TString workdir = homedir + fGridWorkingDir;
923e2ca5 432 if (DirectoryExists(workdir)) {
433 gGrid->Cd(workdir);
434 return;
435 }
436 // Work directory not existing - create it
437 gGrid->Cd(homedir);
b93f8109 438 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 439 gGrid->Cd(fGridWorkingDir);
d3b18c4c 440 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
923e2ca5 441 } else {
d3b18c4c 442 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
923e2ca5 443 workdir.Data(), homedir.Data());
444 fGridWorkingDir = "";
445 }
c57f56b7 446}
447
348be253 448//______________________________________________________________________________
449Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
450{
451// Check if file copying is possible.
d3b18c4c 452 if (fProductionMode) return kTRUE;
348be253 453 if (!Connect()) {
454 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
455 return kFALSE;
456 }
d3339be3 457 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
458 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 459 // Check if alien_CLOSE_SE is defined
460 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
461 if (!closeSE.IsNull()) {
462 Info("CheckFileCopy", "Your current close storage is pointing to: \
463 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
464 } else {
465 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
466 }
467 // Check if grid directory exists.
468 if (!DirectoryExists(alienpath)) {
469 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
470 return kFALSE;
471 }
d3339be3 472 TFile f("plugin_test_copy", "RECREATE");
348be253 473 // User may not have write permissions to current directory
474 if (f.IsZombie()) {
475 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
476 gSystem->WorkingDirectory());
477 return kFALSE;
478 }
479 f.Close();
d3339be3 480 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
481 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
482 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 483 \n# 1. Make sure you have write permissions there. If this is the case: \
484 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
485 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
486 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
487 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
488 gSystem->Unlink(f.GetName());
489 return kFALSE;
490 }
491 gSystem->Unlink(f.GetName());
d3339be3 492 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
493 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 494 return kTRUE;
495}
496
c57f56b7 497//______________________________________________________________________________
498Bool_t AliAnalysisAlien::CheckInputData()
499{
500// Check validity of input data. If necessary, create xml files.
d3b18c4c 501 if (fProductionMode) return kTRUE;
a8739e8a 502 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 503 if (!fGridDataDir.Length()) {
504 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
505 return kFALSE;
506 }
507 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 508 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
509 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 510 return kTRUE;
c57f56b7 511 }
512 // Process declared files
a2f5fc01 513 Bool_t isCollection = kFALSE;
514 Bool_t isXml = kFALSE;
515 Bool_t useTags = kFALSE;
c57f56b7 516 Bool_t checked = kFALSE;
d3b18c4c 517 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 518 TString file;
519 TString workdir = gGrid->GetHomeDirectory();
520 workdir += fGridWorkingDir;
521 if (fInputFiles) {
522 TObjString *objstr;
523 TIter next(fInputFiles);
524 while ((objstr=(TObjString*)next())) {
525 file = workdir;
526 file += "/";
527 file += objstr->GetString();
528 // Store full lfn path
529 if (FileExists(file)) objstr->SetString(file);
530 else {
531 file = objstr->GetName();
532 if (!FileExists(objstr->GetName())) {
533 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
534 objstr->GetName(), workdir.Data());
535 return kFALSE;
536 }
537 }
538 Bool_t iscoll, isxml, usetags;
539 CheckDataType(file, iscoll, isxml, usetags);
540 if (!checked) {
541 checked = kTRUE;
a2f5fc01 542 isCollection = iscoll;
543 isXml = isxml;
544 useTags = usetags;
545 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 546 } else {
a2f5fc01 547 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 548 Error("CheckInputData", "Some conflict was found in the types of inputs");
549 return kFALSE;
550 }
551 }
552 }
553 }
554 // Process requested run numbers
a8739e8a 555 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 556 // Check validity of alien data directory
557 if (!fGridDataDir.Length()) {
558 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
559 return kFALSE;
560 }
923e2ca5 561 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 562 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
563 return kFALSE;
564 }
a2f5fc01 565 if (isCollection) {
c57f56b7 566 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
567 return kFALSE;
568 }
569
a2f5fc01 570 if (checked && !isXml) {
c57f56b7 571 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
572 return kFALSE;
573 }
574 // Check validity of run number(s)
575 TObjArray *arr;
576 TObjString *os;
319593fb 577 Int_t nruns = 0;
904f9f5f 578 TString schunk, schunk2;
c57f56b7 579 TString path;
580 if (!checked) {
581 checked = kTRUE;
a2f5fc01 582 useTags = fDataPattern.Contains("tag");
583 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 584 }
a2f5fc01 585 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 586 Error("CheckInputData", "Cannot mix input files using/not using tags");
587 return kFALSE;
588 }
589 if (fRunNumbers.Length()) {
a8739e8a 590 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 591 arr = fRunNumbers.Tokenize(" ");
592 TIter next(arr);
593 while ((os=(TObjString*)next())) {
594 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 595 if (!DirectoryExists(path)) {
596 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 597 continue;
c57f56b7 598 }
599 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
600 TString msg = "\n##### file: ";
601 msg += path;
602 msg += " type: xml_collection;";
a2f5fc01 603 if (useTags) msg += " using_tags: Yes";
c57f56b7 604 else msg += " using_tags: No";
23329835 605 Info("CheckDataType", "%s", msg.Data());
319593fb 606 if (fNrunsPerMaster<2) {
d2a409b2 607 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 608 } else {
609 nruns++;
610 if (((nruns-1)%fNrunsPerMaster) == 0) {
611 schunk = os->GetString();
612 }
613 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
614 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 615 AddDataFile(schunk);
319593fb 616 }
c57f56b7 617 }
618 delete arr;
a8739e8a 619 } else {
620 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
621 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 622 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 623 if (!DirectoryExists(path)) {
624// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 625 continue;
626 }
5fce53f4 627 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 628 TString msg = "\n##### file: ";
629 msg += path;
630 msg += " type: xml_collection;";
a2f5fc01 631 if (useTags) msg += " using_tags: Yes";
a8739e8a 632 else msg += " using_tags: No";
23329835 633 Info("CheckDataType", "%s", msg.Data());
319593fb 634 if (fNrunsPerMaster<2) {
5fce53f4 635 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 636 } else {
637 nruns++;
638 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 639 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 640 }
904f9f5f 641 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 642 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 643 schunk += schunk2;
d2a409b2 644 AddDataFile(schunk);
319593fb 645 }
a8739e8a 646 }
904f9f5f 647 if (!fInputFiles) {
648 schunk += schunk2;
649 AddDataFile(schunk);
650 }
c57f56b7 651 }
652 return kTRUE;
653}
654
655//______________________________________________________________________________
656Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
657{
658// Create dataset for the grid data directory + run number.
d3b18c4c 659 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 660 if (!Connect()) {
661 Error("CreateDataset", "Cannot create dataset with no grid connection");
662 return kFALSE;
663 }
664
665 // Cd workspace
d3b18c4c 666 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 667 TString workdir = gGrid->GetHomeDirectory();
668 workdir += fGridWorkingDir;
669
670 // Compose the 'find' command arguments
671 TString command;
672 TString options = "-x collection ";
bb885a9e 673 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 674 TString conditions = "";
675
676 TString file;
677 TString path;
319593fb 678 Int_t nruns = 0;
904f9f5f 679 TString schunk, schunk2;
ab254fd1 680 TGridCollection *cbase=0, *cadd=0;
d2a409b2 681 if (!fRunNumbers.Length() && !fRunRange[0]) {
682 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
683 // Make a single data collection from data directory.
684 path = fGridDataDir;
923e2ca5 685 if (!DirectoryExists(path)) {
d2a409b2 686 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
687 return kFALSE;
688 }
923e2ca5 689// CdWork();
d2a409b2 690 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
691 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 692 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 693 command = "find ";
694 command += options;
695 command += path;
696 command += " ";
697 command += pattern;
698 command += conditions;
84fcd93f 699 printf("command: %s\n", command.Data());
d2a409b2 700 TGridResult *res = gGrid->Command(command);
701 if (res) delete res;
702 // Write standard output to file
703 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 704 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
705 Bool_t nullFile = kFALSE;
706 if (!hasGrep) {
d3339be3 707 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
708 } else {
a2f5fc01 709 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
710 if (nullFile) {
defd7a3a 711 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
712 return kFALSE;
713 }
d3339be3 714 }
0f389141 715 }
716 Bool_t fileExists = FileExists(file);
717 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 718 // Copy xml file to alien space
0f389141 719 if (fileExists) gGrid->Rm(file);
d2a409b2 720 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
721 if (!FileExists(file)) {
722 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
723 return kFALSE;
724 }
725 // Update list of files to be processed.
726 }
727 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
728 return kTRUE;
729 }
c57f56b7 730 // Several runs
a2f5fc01 731 Bool_t nullResult = kTRUE;
a8739e8a 732 if (fRunNumbers.Length()) {
733 TObjArray *arr = fRunNumbers.Tokenize(" ");
734 TObjString *os;
735 TIter next(arr);
736 while ((os=(TObjString*)next())) {
737 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 738 if (!DirectoryExists(path)) continue;
739// CdWork();
a8739e8a 740 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
741 else file = Form("%s.xml", os->GetString().Data());
319593fb 742 // If local collection file does not exist, create it via 'find' command.
d3339be3 743 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 744 command = "find ";
745 command += options;
746 command += path;
747 command += pattern;
748 command += conditions;
749 TGridResult *res = gGrid->Command(command);
750 if (res) delete res;
751 // Write standard output to file
752 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 753 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
754 Bool_t nullFile = kFALSE;
755 if (!hasGrep) {
d3339be3 756 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
757 } else {
a2f5fc01 758 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
759 if (nullFile) {
defd7a3a 760 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
761 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
762 continue;
763 }
d3339be3 764 }
a2f5fc01 765 nullResult = kFALSE;
d3339be3 766 }
a8739e8a 767 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 768 // Check if there is one run per master job.
769 if (fNrunsPerMaster<2) {
770 if (FileExists(file)) {
0f389141 771 if (fOverwriteMode) gGrid->Rm(file);
772 else {
773 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
774 continue;
775 }
319593fb 776 }
777 // Copy xml file to alien space
778 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
779 if (!FileExists(file)) {
780 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
781 delete arr;
782 return kFALSE;
783 }
784 } else {
785 nruns++;
786 if (((nruns-1)%fNrunsPerMaster) == 0) {
787 schunk = os->GetString();
788 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
789 } else {
790 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 791 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 792 cbase->Add(cadd);
793 delete cadd;
794 }
795 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
796 continue;
797 }
798 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 799 if (FileExists(schunk)) {
800 if (fOverwriteMode) gGrid->Rm(file);
801 else {
802 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
803 continue;
804 }
319593fb 805 }
84fcd93f 806 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 807 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 808 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 809 if (!FileExists(schunk)) {
810 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
811 delete arr;
812 return kFALSE;
813 }
d3339be3 814 }
a8739e8a 815 }
816 delete arr;
a2f5fc01 817 if (nullResult) {
d3339be3 818 Error("CreateDataset", "No valid dataset corresponding to the query!");
819 return kFALSE;
820 }
a8739e8a 821 } else {
822 // Process a full run range.
823 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 824 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 825 if (!DirectoryExists(path)) continue;
826// CdWork();
a8739e8a 827 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 828 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
0f389141 829 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
830 if (fOverwriteMode) gGrid->Rm(file);
831 else {
832 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
833 continue;
834 }
a8739e8a 835 }
319593fb 836 // If local collection file does not exist, create it via 'find' command.
d3339be3 837 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 838 command = "find ";
839 command += options;
840 command += path;
841 command += pattern;
842 command += conditions;
843 TGridResult *res = gGrid->Command(command);
844 if (res) delete res;
845 // Write standard output to file
846 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 847 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
848 Bool_t nullFile = kFALSE;
849 if (!hasGrep) {
d3339be3 850 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
851 } else {
a2f5fc01 852 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
853 if (nullFile) {
defd7a3a 854 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
855 continue;
856 }
d3339be3 857 }
a2f5fc01 858 nullResult = kFALSE;
319593fb 859 }
a8739e8a 860 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 861 // Check if there is one run per master job.
862 if (fNrunsPerMaster<2) {
863 if (FileExists(file)) {
0f389141 864 if (fOverwriteMode) gGrid->Rm(file);
865 else {
866 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
867 continue;
868 }
319593fb 869 }
870 // Copy xml file to alien space
871 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
872 if (!FileExists(file)) {
873 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
874 return kFALSE;
875 }
876 } else {
877 nruns++;
95e5b448 878 // Check if the collection for the chunk exist locally.
879 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 880 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
881 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
882 else continue;
883 }
84fcd93f 884 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 885 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 886 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 887 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
888 } else {
889 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
890 cbase->Add(cadd);
891 delete cadd;
892 }
904f9f5f 893 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
894 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 895 continue;
896 }
904f9f5f 897 schunk = schunk2;
319593fb 898 if (FileExists(schunk)) {
0f389141 899 if (fOverwriteMode) gGrid->Rm(schunk);
900 else {
901 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
902 continue;
903 }
319593fb 904 }
84fcd93f 905 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 906 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 907 if (FileExists(schunk)) {
0f389141 908 if (fOverwriteMode) gGrid->Rm(schunk);
909 else {
910 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
911 continue;
912 }
95e5b448 913 }
319593fb 914 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
915 if (!FileExists(schunk)) {
916 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
917 return kFALSE;
918 }
919 }
c57f56b7 920 }
a2f5fc01 921 if (nullResult) {
d3339be3 922 Error("CreateDataset", "No valid dataset corresponding to the query!");
923 return kFALSE;
924 }
a8739e8a 925 }
c57f56b7 926 return kTRUE;
927}
928
929//______________________________________________________________________________
930Bool_t AliAnalysisAlien::CreateJDL()
931{
932// Generate a JDL file according to current settings. The name of the file is
933// specified by fJDLName.
934 Bool_t error = kFALSE;
935 TObjArray *arr = 0;
936 Bool_t copy = kTRUE;
d3b18c4c 937 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 938 Bool_t generate = kTRUE;
939 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
940 if (!Connect()) {
941 Error("CreateJDL", "Alien connection required");
942 return kFALSE;
943 }
944 // Check validity of alien workspace
d3b18c4c 945 TString workdir;
946 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
947 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
c57f56b7 948 workdir += fGridWorkingDir;
949 if (generate) {
950 TObjString *os;
951 if (!fInputFiles) {
952 Error("CreateJDL()", "Define some input files for your analysis.");
953 error = kTRUE;
954 }
955 // Compose list of input files
956 // Check if output files were defined
957 if (!fOutputFiles.Length()) {
958 Error("CreateJDL", "You must define at least one output file");
959 error = kTRUE;
960 }
961 // Check if an output directory was defined and valid
962 if (!fGridOutputDir.Length()) {
963 Error("CreateJDL", "You must define AliEn output directory");
964 error = kTRUE;
965 } else {
d3b18c4c 966 if (!fProductionMode) {
967 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
968 if (!DirectoryExists(fGridOutputDir)) {
969 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
970 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
971 } else {
972 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
973 // error = kTRUE;
974 }
975 }
976 gGrid->Cd(workdir);
977 }
c57f56b7 978 }
979 // Exit if any error up to now
980 if (error) return kFALSE;
981 // Set JDL fields
0f389141 982 if (!fUser.IsNull()) {
983 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
984 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
985 }
986 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
987 TString mergeExec = fExecutable;
988 mergeExec.ReplaceAll(".sh", "_merge.sh");
989 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
990 mergeExec.ReplaceAll(".sh", ".C");
991 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 992 if (!fArguments.IsNull())
993 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
d3b18c4c 994 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
995 else fMergingJDL->SetArguments("$1 $2 $3");
1f0d1ca2 996 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
997 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
998 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
999 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1000
0f389141 1001 if (fMaxInitFailed > 0) {
c57f56b7 1002 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 1003 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1004 }
1005 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 1006 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 1007 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
1008 }
1009 if (fSplitMode.Length()) {
c57f56b7 1010 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 1011 fGridJDL->SetDescription("Split", "We split per SE or file");
1012 }
1013 if (!fAliROOTVersion.IsNull()) {
1014 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1015 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1016 }
1017 if (!fROOTVersion.IsNull()) {
c57f56b7 1018 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1019 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1020 }
1021 if (!fAPIVersion.IsNull()) {
c57f56b7 1022 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1023 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1024 }
648174cf 1025 if (!fExternalPackages.IsNull()) {
1026 arr = fExternalPackages.Tokenize(" ");
1027 TIter next(arr);
1028 while ((os=(TObjString*)next())) {
1029 TString pkgname = os->GetString();
1030 Int_t index = pkgname.Index("::");
1031 TString pkgversion = pkgname(index+2, pkgname.Length());
1032 pkgname.Remove(index);
1033 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1034 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1035 }
1036 delete arr;
1037 }
0f389141 1038 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1039 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1040 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1041 TString analysisFile = fExecutable;
1042 analysisFile.ReplaceAll(".sh", ".root");
1043 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1044 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1045 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1046 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1047 if (fAdditionalLibs.Length()) {
1048 arr = fAdditionalLibs.Tokenize(" ");
1049 TIter next(arr);
1050 while ((os=(TObjString*)next())) {
1051 if (os->GetString().Contains(".so")) continue;
1052 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1053 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1054 }
1055 delete arr;
1056 }
4e5c5506 1057 if (fPackages) {
1058 TIter next(fPackages);
1059 TObject *obj;
0f389141 1060 while ((obj=next())) {
4e5c5506 1061 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1062 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1063 }
4e5c5506 1064 }
c57f56b7 1065 if (fOutputArchive.Length()) {
1066 arr = fOutputArchive.Tokenize(" ");
1067 TIter next(arr);
0f389141 1068 Bool_t first = kTRUE;
1069 const char *comment = "Files to be archived";
1070 const char *comment1 = comment;
1071 while ((os=(TObjString*)next())) {
1072 if (!first) comment = NULL;
1073 if (!os->GetString().Contains("@") && fCloseSE.Length())
1074 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1075 else
1076 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1077 first = kFALSE;
1078 }
c57f56b7 1079 delete arr;
0f389141 1080 TString outputArchive = fOutputArchive;
1081 if (!fMergeExcludes.IsNull()) {
1082 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1083 TIter next1(arr);
1084 while ((os=(TObjString*)next1())) {
0f389141 1085 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1086 outputArchive.ReplaceAll(os->GetString(),"");
1087 }
1088 delete arr;
1089 }
d3b18c4c 1090 if (!fTerminateFiles.IsNull()) {
1091 fTerminateFiles.Strip();
1092 fTerminateFiles.ReplaceAll(" ", ",");
1093 outputArchive.ReplaceAll("root_archive.zip:", Form("root_archive.zip:%s,", fTerminateFiles.Data()));
1094 }
0f389141 1095 arr = outputArchive.Tokenize(" ");
705adb3e 1096 TIter next2(arr);
0f389141 1097 comment = comment1;
1098 first = kTRUE;
705adb3e 1099 while ((os=(TObjString*)next2())) {
0f389141 1100 if (!first) comment = NULL;
0cdf65a8 1101 TString currentfile = os->GetString();
1102 currentfile.ReplaceAll(".root", "*.root");
d3b18c4c 1103 if (!IsOneStageMerging()) currentfile.ReplaceAll(".zip", "-Stage$2_$3.zip");
0cdf65a8 1104 if (!currentfile.Contains("@") && fCloseSE.Length())
d3b18c4c 1105 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1106 else
0cdf65a8 1107 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1108 first = kFALSE;
1109 }
1110 delete arr;
c57f56b7 1111 }
149d288c 1112 arr = fOutputFiles.Tokenize(",");
c57f56b7 1113 TIter next(arr);
0f389141 1114 Bool_t first = kTRUE;
1115 const char *comment = "Files to be archived";
1116 const char *comment1 = comment;
43da816a 1117 while ((os=(TObjString*)next())) {
e1eaf596 1118 // Ignore ouputs in jdl that are also in outputarchive
1119 TString sout = os->GetString();
1120 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1121 if (fOutputArchive.Contains(sout)) continue;
0f389141 1122 if (!first) comment = NULL;
43da816a 1123 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1124 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1125 else
0f389141 1126 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
1127 first = kFALSE;
43da816a 1128 }
c57f56b7 1129 delete arr;
0f389141 1130 if (fOutputFiles.Length()) {
1131 TString outputFiles = fOutputFiles;
1132 if (!fMergeExcludes.IsNull()) {
1133 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1134 TIter next1(arr);
1135 while ((os=(TObjString*)next1())) {
0f389141 1136 outputFiles.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1137 outputFiles.ReplaceAll(os->GetString(),"");
1138 }
1139 delete arr;
1140 }
1141 arr = outputFiles.Tokenize(" ");
705adb3e 1142 TIter next2(arr);
0f389141 1143 comment = comment1;
1144 first = kTRUE;
705adb3e 1145 while ((os=(TObjString*)next2())) {
0f389141 1146 // Ignore ouputs in jdl that are also in outputarchive
1147 TString sout = os->GetString();
1148 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1149 if (fOutputArchive.Contains(sout)) continue;
1150 if (!first) comment = NULL;
1151 if (!os->GetString().Contains("@") && fCloseSE.Length())
1152 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1153 else
1154 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
020d7c04 1155 first = kFALSE;
0f389141 1156 }
1157 delete arr;
1158 }
1159 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1160 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
d3b18c4c 1161 TString validationScript = fValidationScript;
0f389141 1162 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
d3b18c4c 1163 validationScript.ReplaceAll(".sh", "_merge.sh");
0f389141 1164 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1165 if (fMasterResubmitThreshold) {
1166 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1167 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1168 }
d2a409b2 1169 // Write a jdl with 2 input parameters: collection name and output dir name.
1170 WriteJDL(copy);
c57f56b7 1171 }
1172 // Copy jdl to grid workspace
a8739e8a 1173 if (copy) {
b5e4aaa7 1174 // Check if an output directory was defined and valid
1175 if (!fGridOutputDir.Length()) {
1176 Error("CreateJDL", "You must define AliEn output directory");
1177 return kFALSE;
1178 } else {
1179 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1180 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1181 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1182 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1183 } else {
1184 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1185 return kFALSE;
1186 }
1187 }
1188 gGrid->Cd(workdir);
1189 }
648174cf 1190 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1191 TString mergeJDLName = fExecutable;
1192 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1193 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1194 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1195 if (fProductionMode) {
648174cf 1196 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1197 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1198 }
648174cf 1199 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1200 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1201 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1202 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1203 if (fMergeViaJDL) {
1204 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1205 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1206 }
648174cf 1207 }
c57f56b7 1208 if (fAdditionalLibs.Length()) {
e7c71df0 1209 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1210 TObjString *os;
1211 TIter next(arr);
1212 while ((os=(TObjString*)next())) {
c57f56b7 1213 if (os->GetString().Contains(".so")) continue;
4e5c5506 1214 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1215 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1216 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1217 }
1218 delete arr;
1219 }
4e5c5506 1220 if (fPackages) {
1221 TIter next(fPackages);
1222 TObject *obj;
1223 while ((obj=next())) {
fdbbc7be 1224 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1225 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1226 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1227 }
1228 }
c57f56b7 1229 }
1230 return kTRUE;
1231}
1232
a8739e8a 1233//______________________________________________________________________________
d2a409b2 1234Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1235{
1236// Writes one or more JDL's corresponding to findex. If findex is negative,
1237// all run numbers are considered in one go (jdl). For non-negative indices
1238// they correspond to the indices in the array fInputFiles.
1239 if (!fInputFiles) return kFALSE;
d3b18c4c 1240 TObject *os;
1241 TString workdir;
1242 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
a8739e8a 1243 workdir += fGridWorkingDir;
d2a409b2 1244
d3b18c4c 1245 if (fProductionMode) {
a8739e8a 1246 TIter next(fInputFiles);
d3b18c4c 1247 while ((os=next()))
1248 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1249 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1250 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1251 } else {
1252 if (!fRunNumbers.Length() && !fRunRange[0]) {
1253 // One jdl with no parameters in case input data is specified by name.
1254 TIter next(fInputFiles);
1255 while ((os=next()))
1256 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1257 if (!fOutputSingle.IsNull())
1258 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1259 else {
1260 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1261 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1262 }
1263 } else {
1264 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1265 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1266 if (!fOutputSingle.IsNull()) {
1267 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1268 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1269 } else {
1270 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1271 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1272 }
1273 }
a8739e8a 1274 }
1275
a8739e8a 1276 // Generate the JDL as a string
1277 TString sjdl = fGridJDL->Generate();
0f389141 1278 TString sjdl1 = fMergingJDL->Generate();
a8739e8a 1279 Int_t index;
a8739e8a 1280 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1281 sjdl.ReplaceAll("(member", "\n (member");
1282 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1283 sjdl.ReplaceAll("{", "{\n ");
1284 sjdl.ReplaceAll("};", "\n};");
1285 sjdl.ReplaceAll("{\n \n", "{\n");
1286 sjdl.ReplaceAll("\n\n", "\n");
1287 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1288 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1289 sjdl1.ReplaceAll("(member", "\n (member");
1290 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1291 sjdl1.ReplaceAll("{", "{\n ");
1292 sjdl1.ReplaceAll("};", "\n};");
1293 sjdl1.ReplaceAll("{\n \n", "{\n");
1294 sjdl1.ReplaceAll("\n\n", "\n");
1295 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1296 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1297 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1298 index = sjdl.Index("JDLVariables");
1299 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1300 sjdl += "Workdirectorysize = {\"5000MB\"};";
0f389141 1301 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
d3b18c4c 1302 index = fJobTag.Index(":");
1303 if (index < 0) index = fJobTag.Length();
1304 TString jobTag = fJobTag;
1305 jobTag.Insert(index, "_Merging");
1306 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
7c2cd90a 1307 sjdl1.Prepend("# Generated merging jdl\n# $1 = full alien path to output directory to be merged\n# $2 = merging stage\n# $3 = merged chunk\n");
0f389141 1308 index = sjdl1.Index("JDLVariables");
1309 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
4739b2aa 1310 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
a8739e8a 1311 // Write jdl to file
a8739e8a 1312 ofstream out;
d2a409b2 1313 out.open(fJDLName.Data(), ios::out);
a8739e8a 1314 if (out.bad()) {
d3b18c4c 1315 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1316 return kFALSE;
1317 }
1318 out << sjdl << endl;
0f389141 1319 TString mergeJDLName = fExecutable;
1320 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1321 if (fMergeViaJDL) {
1322 ofstream out1;
1323 out1.open(mergeJDLName.Data(), ios::out);
1324 if (out.bad()) {
d3b18c4c 1325 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
0f389141 1326 return kFALSE;
1327 }
1328 out1 << sjdl1 << endl;
1329 }
a8739e8a 1330
1331 // Copy jdl to grid workspace
1332 if (!copy) {
d3b18c4c 1333 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
a8739e8a 1334 } else {
d2a409b2 1335 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1336 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1337 if (fProductionMode) {
b5fe9cba 1338 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1339 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1340 }
d2a409b2 1341 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1342 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
d3b18c4c 1343 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1344 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1345 if (fMergeViaJDL) {
d3b18c4c 1346 Info("WriteJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
0f389141 1347 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1348 }
a8739e8a 1349 }
1350 return kTRUE;
1351}
1352
c57f56b7 1353//______________________________________________________________________________
5513444a 1354Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1355{
1356// Returns true if file exists.
5513444a 1357 if (!gGrid) return kFALSE;
c57f56b7 1358 TGridResult *res = gGrid->Ls(lfn);
1359 if (!res) return kFALSE;
1360 TMap *map = dynamic_cast<TMap*>(res->At(0));
1361 if (!map) {
1362 delete res;
1363 return kFALSE;
1364 }
1365 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1366 if (!objs || !objs->GetString().Length()) {
1367 delete res;
1368 return kFALSE;
1369 }
1370 delete res;
1371 return kTRUE;
1372}
1373
923e2ca5 1374//______________________________________________________________________________
1375Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1376{
1377// Returns true if directory exists. Can be also a path.
1378 if (!gGrid) return kFALSE;
1379 // Check if dirname is a path
1380 TString dirstripped = dirname;
1381 dirstripped = dirstripped.Strip();
1382 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1383 TString dir = gSystem->BaseName(dirstripped);
1384 dir += "/";
1385 TString path = gSystem->DirName(dirstripped);
1386 TGridResult *res = gGrid->Ls(path, "-F");
1387 if (!res) return kFALSE;
1388 TIter next(res);
1389 TMap *map;
1390 TObject *obj;
1391 while ((map=dynamic_cast<TMap*>(next()))) {
1392 obj = map->GetValue("name");
1393 if (!obj) break;
1394 if (dir == obj->GetName()) {
1395 delete res;
1396 return kTRUE;
1397 }
1398 }
1399 delete res;
1400 return kFALSE;
1401}
1402
c57f56b7 1403//______________________________________________________________________________
a2f5fc01 1404void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1405{
1406// Check input data type.
a2f5fc01 1407 isCollection = kFALSE;
1408 isXml = kFALSE;
1409 useTags = kFALSE;
c57f56b7 1410 if (!gGrid) {
1411 Error("CheckDataType", "No connection to grid");
1412 return;
1413 }
a2f5fc01 1414 isCollection = IsCollection(lfn);
c57f56b7 1415 TString msg = "\n##### file: ";
1416 msg += lfn;
a2f5fc01 1417 if (isCollection) {
c57f56b7 1418 msg += " type: raw_collection;";
1419 // special treatment for collections
a2f5fc01 1420 isXml = kFALSE;
c57f56b7 1421 // check for tag files in the collection
1422 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1423 if (!res) {
1424 msg += " using_tags: No (unknown)";
23329835 1425 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1426 return;
1427 }
1428 const char* typeStr = res->GetKey(0, "origLFN");
1429 if (!typeStr || !strlen(typeStr)) {
1430 msg += " using_tags: No (unknown)";
23329835 1431 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1432 return;
1433 }
1434 TString file = typeStr;
a2f5fc01 1435 useTags = file.Contains(".tag");
1436 if (useTags) msg += " using_tags: Yes";
c57f56b7 1437 else msg += " using_tags: No";
23329835 1438 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1439 return;
1440 }
1441 TString slfn(lfn);
1442 slfn.ToLower();
a2f5fc01 1443 isXml = slfn.Contains(".xml");
1444 if (isXml) {
c57f56b7 1445 // Open xml collection and check if there are tag files inside
1446 msg += " type: xml_collection;";
1447 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1448 if (!coll) {
1449 msg += " using_tags: No (unknown)";
23329835 1450 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1451 return;
1452 }
1453 TMap *map = coll->Next();
1454 if (!map) {
1455 msg += " using_tags: No (unknown)";
23329835 1456 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1457 return;
1458 }
1459 map = (TMap*)map->GetValue("");
1460 TString file;
1461 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1462 useTags = file.Contains(".tag");
c57f56b7 1463 delete coll;
a2f5fc01 1464 if (useTags) msg += " using_tags: Yes";
c57f56b7 1465 else msg += " using_tags: No";
23329835 1466 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1467 return;
1468 }
a2f5fc01 1469 useTags = slfn.Contains(".tag");
c57f56b7 1470 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1471 else msg += " type: unknown file;";
a2f5fc01 1472 if (useTags) msg += " using_tags: Yes";
c57f56b7 1473 else msg += " using_tags: No";
23329835 1474 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1475}
1476
4e5c5506 1477//______________________________________________________________________________
1478void AliAnalysisAlien::EnablePackage(const char *package)
1479{
1480// Enables a par file supposed to exist in the current directory.
1481 TString pkg(package);
1482 pkg.ReplaceAll(".par", "");
1483 pkg += ".par";
1484 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1485 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1486 return;
1487 }
1488 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1489 Info("EnablePackage", "AliEn plugin will use .par packages");
1490 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1491 if (!fPackages) {
1492 fPackages = new TObjArray();
1493 fPackages->SetOwner();
1494 }
1495 fPackages->Add(new TObjString(pkg));
1496}
1497
3bdcb562 1498//______________________________________________________________________________
1499TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1500{
1501// Make a tree from files having the location specified in fFileForTestMode.
1502// Inspired from JF's CreateESDChain.
1503 if (fFileForTestMode.IsNull()) {
1504 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1505 return NULL;
1506 }
1507 if (gSystem->AccessPathName(fFileForTestMode)) {
1508 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1509 return NULL;
1510 }
1511 // Open the file
1512 ifstream in;
1513 in.open(fFileForTestMode);
1514 Int_t count = 0;
1515 // Read the input list of files and add them to the chain
1516 TString line;
1517 TChain *chain = new TChain(treeName);
1518 while (in.good())
1519 {
1520 in >> line;
1521 if (line.IsNull()) continue;
1522 if (count++ == fNtestFiles) break;
1523 TString esdFile(line);
1524 TFile *file = TFile::Open(esdFile);
1525 if (file) {
1526 if (!file->IsZombie()) chain->Add(esdFile);
1527 file->Close();
1528 } else {
1529 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1530 }
1531 }
1532 in.close();
1533 if (!chain->GetListOfFiles()->GetEntries()) {
1534 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1535 delete chain;
1536 return NULL;
1537 }
1538// chain->ls();
1539 return chain;
1540}
1541
d2a409b2 1542//______________________________________________________________________________
1543const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1544{
1545// Get job status for all jobs with jobid>jobidstart.
1546 static char mstatus[20];
1547 mstatus[0] = '\0';
1548 nrunning = 0;
1549 nwaiting = 0;
1550 nerror = 0;
1551 ndone = 0;
1552 TGridJobStatusList *list = gGrid->Ps("");
1553 if (!list) return mstatus;
1554 Int_t nentries = list->GetSize();
1555 TGridJobStatus *status;
1556 Int_t pid;
1557 for (Int_t ijob=0; ijob<nentries; ijob++) {
1558 status = (TGridJobStatus *)list->At(ijob);
1559 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1560 if (pid<jobidstart) continue;
1561 if (pid == lastid) {
1562 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1563 }
1564 switch (status->GetStatus()) {
1565 case TGridJobStatus::kWAITING:
1566 nwaiting++; break;
1567 case TGridJobStatus::kRUNNING:
1568 nrunning++; break;
1569 case TGridJobStatus::kABORTED:
1570 case TGridJobStatus::kFAIL:
1571 case TGridJobStatus::kUNKNOWN:
1572 nerror++; break;
1573 case TGridJobStatus::kDONE:
1574 ndone++;
1575 }
1576 }
1577 list->Delete();
1578 delete list;
1579 return mstatus;
1580}
1581
c57f56b7 1582//______________________________________________________________________________
1583Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1584{
1585// Returns true if file is a collection. Functionality duplicated from
1586// TAlien::Type() because we don't want to directly depend on TAlien.
1587 if (!gGrid) {
1588 Error("IsCollection", "No connection to grid");
1589 return kFALSE;
1590 }
1591 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1592 if (!res) return kFALSE;
1593 const char* typeStr = res->GetKey(0, "type");
1594 if (!typeStr || !strlen(typeStr)) return kFALSE;
1595 if (!strcmp(typeStr, "collection")) return kTRUE;
1596 delete res;
1597 return kFALSE;
1598}
1599
fe2d7fc2 1600//______________________________________________________________________________
1601Bool_t AliAnalysisAlien::IsSingleOutput() const
1602{
1603// Check if single-ouput option is on.
1604 return (!fOutputSingle.IsNull());
1605}
1606
16a4353c 1607//______________________________________________________________________________
1608void AliAnalysisAlien::Print(Option_t *) const
1609{
1610// Print current plugin settings.
84fcd93f 1611 printf("### AliEn analysis plugin current settings ###\n");
d3b18c4c 1612 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1613 if (mgr && mgr->IsProofMode()) {
1614 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1615 if (TestBit(AliAnalysisGrid::kTest))
1616 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1617 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1618 if (!fProofDataSet.IsNull())
1619 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1620 if (fProofReset==1)
1621 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1622 if (fProofReset>1)
1623 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1624 if (!fRootVersionForProof.IsNull())
1625 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1626 else
1627 printf("= ROOT version requested________________________ default\n");
1628 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1629 if (!fAliRootMode.IsNull())
1630 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1631 if (fNproofWorkers)
1632 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1633 if (fNproofWorkersPerSlave)
1634 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1635 if (TestSpecialBit(kClearPackages))
1636 printf("= ClearPackages requested...\n");
1637 if (fIncludePath.Data())
1638 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1639 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1640 if (fPackages && fPackages->GetEntries()) {
1641 TIter next(fPackages);
1642 TObject *obj;
1643 TString list;
1644 while ((obj=next())) list += obj->GetName();
1645 printf("= Par files to be used: ________________________ %s\n", list.Data());
1646 }
1647 if (TestSpecialBit(kProofConnectGrid))
1648 printf("= Requested PROOF connection to grid\n");
1649 return;
1650 }
e1c22e21 1651 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1652 if (fOverwriteMode) {
1653 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1654 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1655 }
348be253 1656 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1657 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1658 printf("= Production mode:______________________________ %d\n", fProductionMode);
1659 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1660 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1661 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1662 if (fUser.Length())
84fcd93f 1663 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1664 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1665 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1666 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1667 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1668 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1669 if (fRunNumbers.Length())
84fcd93f 1670 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1671 if (fRunRange[0])
5fce53f4 1672 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1673 if (!fRunRange[0] && !fRunNumbers.Length()) {
1674 TIter next(fInputFiles);
1675 TObject *obj;
1676 TString list;
1677 while ((obj=next())) list += obj->GetName();
84fcd93f 1678 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1679 }
1680 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1681 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1682 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1683 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1684 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
d3b18c4c 1685 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
84fcd93f 1686 printf("=====================================================================\n");
1687 printf("= Job price: ___________________________________ %d\n", fPrice);
1688 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1689 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1690 if (fMaxInitFailed>0)
84fcd93f 1691 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1692 if (fMasterResubmitThreshold>0)
84fcd93f 1693 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1694 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1695 if (fNrunsPerMaster>0)
84fcd93f 1696 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1697 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1698 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1699 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1700 if (fArguments.Length())
84fcd93f 1701 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1702 if (fExecutableArgs.Length())
1703 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1704 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1705 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1706 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1707 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1708 if (fDatasetName)
84fcd93f 1709 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1710 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1711 if (fIncludePath.Data())
84fcd93f 1712 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1713 if (fCloseSE.Length())
84fcd93f 1714 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1715 if (fFriendChainName.Length())
84fcd93f 1716 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
d3b18c4c 1717 if (fPackages && fPackages->GetEntries()) {
16a4353c 1718 TIter next(fPackages);
1719 TObject *obj;
1720 TString list;
1721 while ((obj=next())) list += obj->GetName();
84fcd93f 1722 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1723 }
1724}
1725
c57f56b7 1726//______________________________________________________________________________
1727void AliAnalysisAlien::SetDefaults()
1728{
1729// Set default values for everything. What cannot be filled will be left empty.
1730 if (fGridJDL) delete fGridJDL;
1731 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1732 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1733 fPrice = 1;
1734 fTTL = 30000;
1735 fSplitMaxInputFileNumber = 100;
1736 fMaxInitFailed = 0;
1737 fMasterResubmitThreshold = 0;
bb885a9e 1738 fNtestFiles = 10;
149d288c 1739 fNreplicas = 2;
a8739e8a 1740 fRunRange[0] = 0;
1741 fRunRange[1] = 0;
319593fb 1742 fNrunsPerMaster = 1;
16a4353c 1743 fMaxMergeFiles = 100;
c57f56b7 1744 fRunNumbers = "";
1745 fExecutable = "analysis.sh";
0a1c1f7f 1746 fExecutableCommand = "root -b -q";
c57f56b7 1747 fArguments = "";
631c0b05 1748 fExecutableArgs = "";
c57f56b7 1749 fAnalysisMacro = "myAnalysis.C";
1750 fAnalysisSource = "";
1751 fAdditionalLibs = "";
1752 fSplitMode = "se";
1753 fAPIVersion = "";
1754 fROOTVersion = "";
1755 fAliROOTVersion = "";
1756 fUser = ""; // Your alien user name
1757 fGridWorkingDir = "";
1758 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1759 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1760 fFriendChainName = "";
c57f56b7 1761 fGridOutputDir = "output";
149d288c 1762 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1763 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1764 fInputFormat = "xml-single";
1765 fJDLName = "analysis.jdl";
c6cb3634 1766 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1767 fMergeExcludes = "";
0f389141 1768 fMergeViaJDL = 0;
348be253 1769 SetUseCopy(kTRUE);
1770 SetCheckCopy(kTRUE);
149d288c 1771 SetDefaultOutputs(kTRUE);
e1c22e21 1772 fOverwriteMode = 1;
c57f56b7 1773}
1774
0f389141 1775//______________________________________________________________________________
7c2cd90a 1776Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, Bool_t submit, const char *jdl)
0f389141 1777{
7c2cd90a 1778// Static method that checks the status of merging. This can submit merging jobs that did not produced the expected
1779// output. If <submit> is false (checking) returns true only when the final merged file was found. If submit is true returns
1780// true if the jobs were successfully submitted.
1781 Int_t countOrig = 0;
1782 Int_t countStage = 0;
1783 Int_t stage = 0;
1784 Int_t i;
1785 Bool_t doneFinal = kFALSE;
1786 TBits chunksDone;
1787 TString saliendir(aliendir);
1788 TString sfilename, stmp;
1789 saliendir.ReplaceAll("//","/");
1790 saliendir = saliendir.Strip(TString::kTrailing, '/');
1791 if (!gGrid) {
1792 ::Error("GetNregisteredFiles", "You need to be connected to AliEn.");
1793 return kFALSE;
1794 }
0cdf65a8 1795 sfilename = filename;
1796 sfilename.ReplaceAll(".root", "*.root");
1797 printf("Checking directory <%s> for merged files <%s> ...\n", aliendir, sfilename.Data());
1798 TString command = Form("find %s/ *%s", saliendir.Data(), sfilename.Data());
7c2cd90a 1799 TGridResult *res = gGrid->Command(command);
1800 if (!res) {
1801 ::Error("GetNregisteredFiles","Error: No result for the find command\n");
1802 return kFALSE;
1803 }
1804 TIter nextmap(res);
1805 TMap *map = 0;
1806 while ((map=(TMap*)nextmap())) {
1807 TString turl = map->GetValue("turl")->GetName();
1808 if (!turl.Length()) {
1809 // Nothing found
1810 delete res;
1811 return kFALSE;
1812 }
1813 turl.ReplaceAll("alien://", "");
1814 turl.ReplaceAll(saliendir, "");
1815 sfilename = gSystem->BaseName(turl);
0cdf65a8 1816 turl = turl.Strip(TString::kLeading, '/');
7c2cd90a 1817 // Now check to what the file corresponds to:
1818 // original output - aliendir/%03d/filename
1819 // merged file (which stage) - aliendir/filename-Stage%02d_%04d
1820 // final merged file - aliendir/filename
1821 if (sfilename == turl) {
1822 if (sfilename == filename) {
1823 doneFinal = kTRUE;
1824 } else {
1825 // check stage
1826 Int_t index = sfilename.Index("Stage");
1827 if (index<0) continue;
1828 stmp = sfilename(index+5,2);
1829 Int_t istage = atoi(stmp);
1830 stmp = sfilename(index+8,4);
1831 Int_t ijob = atoi(stmp);
1832 if (istage<stage) continue; // Ignore lower stages
1833 if (istage>stage) {
1834 countStage = 0;
1835 chunksDone.ResetAllBits();
1836 stage = istage;
1837 }
1838 countStage++;
1839 chunksDone.SetBitNumber(ijob);
1840 }
1841 } else {
1842 countOrig++;
1843 }
1844 if (doneFinal) {
1845 delete res;
0cdf65a8 1846 printf("=> Removing files from previous stages...\n");
1847 gGrid->Rm(Form("%s/*Stage*.root", aliendir));
7a62d541 1848 for (i=1; i<stage; i++)
1849 gGrid->Rm(Form("%s/*Stage%d*.zip", aliendir, i));
7c2cd90a 1850 return kTRUE;
1851 }
1852 }
1853 delete res;
1854 // Compute number of jobs that were submitted for the current stage
1855 Int_t ntotstage = countOrig;
7ae54d70 1856 for (i=1; i<=stage; i++) {
1857 if (ntotstage%nperchunk) ntotstage = (ntotstage/nperchunk)+1;
1858 else ntotstage = (ntotstage/nperchunk);
1859 }
7c2cd90a 1860 // Now compare with the number of set bits in the chunksDone array
1861 Int_t nmissing = (stage>0)?(ntotstage - countStage):0;
1862 // Print the info
1863 printf("*** Found %d original files\n", countOrig);
1864 if (stage==0) printf("*** No merging completed so far.\n");
1865 else printf("*** Found %d out of %d files merged for stage %d\n", countStage, ntotstage, stage);
1866 if (nmissing) printf("*** Number of merged files missing for this stage: %d -> check merging job completion\n", nmissing);
7c2cd90a 1867 if (!submit) return doneFinal;
1868 // Sumbit merging jobs for all missing chunks for the current stage.
1869 TString query = Form("submit %s %s", jdl, aliendir);
1870 Int_t ichunk = -1;
1871 if (nmissing) {
1872 for (i=0; i<nmissing; i++) {
1873 ichunk = chunksDone.FirstNullBit(ichunk+1);
1874 Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage, ichunk));
1875 if (!jobId) return kFALSE;
1876 }
1877 return kTRUE;
1878 }
1879 // Submit next stage of merging
1880 if (stage==0) countStage = countOrig;
7ae54d70 1881 Int_t nchunks = (countStage/nperchunk);
1882 if (countStage%nperchunk) nchunks += 1;
7c2cd90a 1883 for (i=0; i<nchunks; i++) {
1884 Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage+1, i));
1885 if (!jobId) return kFALSE;
1886 }
1887 return kTRUE;
1888}
1889
1890//______________________________________________________________________________
1891Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1892{
1893// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1894 if (!gGrid) return 0;
1895 printf("=> %s ------> ",query);
1896 TGridResult *res = gGrid->Command(query);
1897 if (!res) return 0;
1898 TString jobId = res->GetKey(0,"jobId");
1899 delete res;
1900 if (jobId.IsNull()) {
1901 printf("submission failed. Reason:\n");
1902 gGrid->Stdout();
1903 gGrid->Stderr();
1904 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1905 return 0;
1906 }
1907 printf(" Job id: %s\n", jobId.Data());
1908 return atoi(jobId);
1909}
1910
1911//______________________________________________________________________________
1912Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage, Int_t ichunk)
1913{
1914// Merge given output files from basedir. The file merger will merge nmaxmerge
1915// files in a group. Merging can be done in stages:
1916// stage=0 : will merge all existing files in a single stage
1917// stage=1 : does a find command for all files that do NOT contain the string "Stage".
1918// If their number is bigger that nmaxmerge, only the files from
1919// ichunk*nmaxmerge to ichunk*(nmaxmerge+1)-1 will get merged as output_stage_<ichunk>
1920// stage=n : does a find command for files named <output>Stage<stage-1>_*. If their number is bigger than
1921// nmaxmerge, merge just the chunk ichunk, otherwise write the merged output to the file
1922// named <output>.
a2f5fc01 1923 TString outputFile = output;
0f389141 1924 TString command;
a2f5fc01 1925 TString outputChunk;
1926 TString previousChunk = "";
1927 Int_t countChunk = 0;
1928 Int_t countZero = nmaxmerge;
0f389141 1929 Bool_t merged = kTRUE;
a2f5fc01 1930 Int_t index = outputFile.Index("@");
1931 if (index > 0) outputFile.Remove(index);
7c2cd90a 1932 TString inputFile = outputFile;
1933 if (stage>1) inputFile.ReplaceAll(".root", Form("-Stage%02d_*.root", stage-1));
1934 command = Form("find %s/ *%s", basedir, inputFile.Data());
0f389141 1935 printf("command: %s\n", command.Data());
1936 TGridResult *res = gGrid->Command(command);
1937 if (!res) {
7c2cd90a 1938 ::Error("MergeOutput","No result for the find command\n");
0f389141 1939 return kFALSE;
1940 }
1941
1942 TFileMerger *fm = 0;
1943 TIter nextmap(res);
1944 TMap *map = 0;
7c2cd90a 1945 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 1946 outputChunk = outputFile;
1947 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 1948 // Check for existent temporary merge files
e1c22e21 1949 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 1950 // Preserve old merging functionality for stage 0.
1951 if (stage==0) {
1952 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
1953 while (1) {
1954 // Skip as many input files as in a chunk
1955 for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
1956 if (!map) {
1957 ::Error("MergeOutput", "Cannot resume merging for <%s>, nentries=%d", outputFile.Data(), res->GetSize());
1958 delete res;
1959 return kFALSE;
1960 }
1961 outputChunk = outputFile;
1962 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
1963 countChunk++;
1964 if (gSystem->AccessPathName(outputChunk)) continue;
1965 // Merged file with chunks up to <countChunk> found
1966 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
1967 previousChunk = outputChunk;
1968 break;
1969 }
1970 }
1971 countZero = nmaxmerge;
1972
1973 while ((map=(TMap*)nextmap())) {
1974 // Loop 'find' results and get next LFN
1975 if (countZero == nmaxmerge) {
1976 // First file in chunk - create file merger and add previous chunk if any.
1977 fm = new TFileMerger(kFALSE);
1978 fm->SetFastMethod(kTRUE);
1979 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
1980 outputChunk = outputFile;
1981 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
1982 }
1983 // If last file found, put merged results in the output file
1984 if (map == res->Last()) outputChunk = outputFile;
1985 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1986 if (!objs || !objs->GetString().Length()) {
1987 // Nothing found - skip this output
0f389141 1988 delete res;
7c2cd90a 1989 delete fm;
0f389141 1990 return kFALSE;
7c2cd90a 1991 }
1992 // Add file to be merged and decrement chunk counter.
1993 fm->AddFile(objs->GetString());
1994 countZero--;
1995 if (countZero==0 || map == res->Last()) {
1996 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1997 // Nothing found - skip this output
1998 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
1999 delete res;
2000 delete fm;
2001 return kFALSE;
2002 }
2003 fm->OutputFile(outputChunk);
2004 // Merge the outputs, then go to next chunk
2005 if (!fm->Merge()) {
2006 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2007 delete res;
2008 delete fm;
2009 return kFALSE;
2010 } else {
2011 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2012 gSystem->Unlink(previousChunk);
2013 }
2014 if (map == res->Last()) {
2015 delete res;
2016 delete fm;
2017 break;
2018 }
2019 countChunk++;
2020 countZero = nmaxmerge;
2021 previousChunk = outputChunk;
0f389141 2022 }
0f389141 2023 }
7c2cd90a 2024 return merged;
2025 }
2026 // Merging stage different than 0.
2027 // Move to the begining of the requested chunk.
2028 outputChunk = outputFile;
2029 if (nmaxmerge < res->GetSize()) {
2030 if (ichunk*nmaxmerge >= res->GetSize()) {
2031 ::Error("MergeOutput", "Cannot merge merge chunk %d grouping %d files from %d total.", ichunk, nmaxmerge, res->GetSize());
2032 delete res;
2033 return kFALSE;
2034 }
2035 for (Int_t counter=0; counter<ichunk*nmaxmerge; counter++) map = (TMap*)nextmap();
2036 outputChunk.ReplaceAll(".root", Form("-Stage%02d_%04d.root", stage, ichunk));
2037 }
2038 countZero = nmaxmerge;
2039 fm = new TFileMerger(kFALSE);
2040 fm->SetFastMethod(kTRUE);
0f389141 2041 while ((map=(TMap*)nextmap())) {
7c2cd90a 2042 // Loop 'find' results and get next LFN
0f389141 2043 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2044 if (!objs || !objs->GetString().Length()) {
2045 // Nothing found - skip this output
2046 delete res;
2047 delete fm;
2048 return kFALSE;
2049 }
2050 // Add file to be merged and decrement chunk counter.
2051 fm->AddFile(objs->GetString());
a2f5fc01 2052 countZero--;
7c2cd90a 2053 if (countZero==0) break;
2054 }
2055 delete res;
2056 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2057 // Nothing found - skip this output
2058 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2059 delete fm;
2060 return kFALSE;
2061 }
2062 fm->OutputFile(outputChunk);
2063 // Merge the outputs
2064 if (!fm->Merge()) {
2065 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2066 delete fm;
2067 return kFALSE;
2068 } else {
2069 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
0f389141 2070 }
7c2cd90a 2071 delete fm;
2072 return kTRUE;
0f389141 2073}
2074
c57f56b7 2075//______________________________________________________________________________
2076Bool_t AliAnalysisAlien::MergeOutputs()
2077{
2078// Merge analysis outputs existing in the AliEn space.
2079 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2080 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2081 if (!Connect()) {
2082 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2083 return kFALSE;
0f389141 2084 }
c9e8f7fd 2085 if (fMergeViaJDL) {
2086 if (!TestBit(AliAnalysisGrid::kMerge)) {
2087 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2088 return kFALSE;
2089 }
2090 if (fProductionMode) {
2091 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2092 return kFALSE;
2093 }
0f389141 2094 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2095 if (!SubmitMerging()) return kFALSE;
0f389141 2096 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2097 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2098 return kFALSE;
c57f56b7 2099 }
2100 // Get the output path
d2a409b2 2101 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2102 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2103 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2104 return kFALSE;
2105 }
2106 if (!fOutputFiles.Length()) {
2107 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2108 return kFALSE;
0f389141 2109 }
2110 // Check if fast read option was requested
e1c22e21 2111 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2112 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2113 if (fFastReadOption) {
d3339be3 2114 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2115 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
2116 gEnv->SetValue("XNet.ConnectTimeout",10);
2117 gEnv->SetValue("XNet.RequestTimeout",10);
0f389141 2118 gEnv->SetValue("XNet.MaxRedirectCount",2);
d3339be3 2119 gEnv->SetValue("XNet.ReconnectTimeout",10);
0f389141 2120 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2121 }
e8b839ab 2122 // Make sure we change the temporary directory
2123 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2124 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2125 TIter next(list);
2126 TObjString *str;
a2f5fc01 2127 TString outputFile;
c57f56b7 2128 Bool_t merged = kTRUE;
2129 while((str=(TObjString*)next())) {
a2f5fc01 2130 outputFile = str->GetString();
2131 Int_t index = outputFile.Index("@");
2132 if (index > 0) outputFile.Remove(index);
2133 TString outputChunk = outputFile;
2134 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2135 // Skip already merged outputs
a2f5fc01 2136 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2137 if (fOverwriteMode) {
a2f5fc01 2138 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2139 gSystem->Unlink(outputFile);
2140 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2141 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2142 outputChunk.Data());
2143 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2144 }
2145 } else {
a2f5fc01 2146 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2147 continue;
2148 }
2149 } else {
a2f5fc01 2150 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2151 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2152 outputChunk.Data());
2153 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2154 }
2155 }
bb885a9e 2156 if (fMergeExcludes.Length() &&
a2f5fc01 2157 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2158 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2159 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2160 if (!merged) {
2161 Error("MergeOutputs", "Terminate() will NOT be executed");
2162 return kFALSE;
ff07ec61 2163 }
2164 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2165 if (fileOpened) fileOpened->Close();
c57f56b7 2166 }
0f389141 2167 return kTRUE;
c57f56b7 2168}
2169
bb885a9e 2170//______________________________________________________________________________
2171void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2172{
2173// Use the output files connected to output containers from the analysis manager
2174// rather than the files defined by SetOutputFiles
2175 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2176 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2177 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2178}
2179
149d288c 2180//______________________________________________________________________________
2181void AliAnalysisAlien::SetOutputFiles(const char *list)
2182{
2183// Manually set the output files list.
2184// Removes duplicates. Not allowed if default outputs are not disabled.
2185 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2186 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2187 return;
2188 }
2189 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2190 fOutputFiles = "";
2191 TString slist = list;
2192 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2193 TObjArray *arr = slist.Tokenize(" ");
2194 TObjString *os;
2195 TIter next(arr);
2196 TString sout;
2197 while ((os=(TObjString*)next())) {
2198 sout = os->GetString();
2199 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2200 if (fOutputFiles.Contains(sout)) continue;
2201 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2202 fOutputFiles += sout;
2203 }
2204 delete arr;
2205}
2206
2207//______________________________________________________________________________
2208void AliAnalysisAlien::SetOutputArchive(const char *list)
2209{
2210// Manually set the output archive list. Free text - you are on your own...
2211// Not allowed if default outputs are not disabled.
2212 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2213 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2214 return;
2215 }
2216 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2217 fOutputArchive = list;
2218}
2219
2220//______________________________________________________________________________
2221void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2222{
2223// Setting a prefered output SE is not allowed anymore.
2224 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2225}
2226
5513444a 2227//______________________________________________________________________________
2228Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2229{
2230// Start remote grid analysis.
3bdcb562 2231 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2232 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2233 if (!mgr || !mgr->IsInitialized()) {
2234 Error("StartAnalysis", "You need an initialized analysis manager for this");
2235 return kFALSE;
2236 }
2237 // Are we in PROOF mode ?
2238 if (mgr->IsProofMode()) {
90d50a8c 2239 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3bdcb562 2240 if (fProofCluster.IsNull()) {
2241 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2242 return kFALSE;
2243 }
2244 if (fProofDataSet.IsNull() && !testMode) {
2245 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2246 return kFALSE;
2247 }
2248 // Set the needed environment
2249 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2250 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2251 if (fProofReset && !testMode) {
2252 if (fProofReset==1) {
2253 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2254 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2255 } else {
2256 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2257 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2258 }
2259 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2260 return kFALSE;
2261 }
2262 // Do we need to change the ROOT version ? The success of this cannot be checked.
2263 if (!fRootVersionForProof.IsNull() && !testMode) {
2264 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2265 fProofCluster.Data(), fRootVersionForProof.Data()));
2266 }
2267 // Connect to PROOF and check the status
2268 Long_t proof = 0;
27734f0e 2269 TString sworkers;
2270 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2271 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2272 if (!testMode) {
27734f0e 2273 if (!sworkers.IsNull())
2274 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2275 else
2276 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2277 } else {
2278 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2279 if (!proof) {
2280 Error("StartAnalysis", "Could not start PROOF in test mode");
2281 return kFALSE;
2282 }
2283 }
2284 if (!proof) {
2285 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2286 return kFALSE;
2287 }
27734f0e 2288 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2289 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2290 // Is dataset existing ?
2291 if (!testMode) {
2292 TString dataset = fProofDataSet;
2293 Int_t index = dataset.Index("#");
2294 if (index>=0) dataset.Remove(index);
830acc4c 2295// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2296// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2297// return kFALSE;
2298// }
2299// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2300 }
2301 // Is ClearPackages() needed ?
2302 if (TestSpecialBit(kClearPackages)) {
2303 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2304 gROOT->ProcessLine("gProof->ClearPackages();");
2305 }
2306 // Is a given aliroot mode requested ?
2307 TList optionsList;
830acc4c 2308 if (!fAliRootMode.IsNull()) {
3bdcb562 2309 TString alirootMode = fAliRootMode;
2310 if (alirootMode == "default") alirootMode = "";
2311 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2312 optionsList.SetOwner();
2313 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2314 // Check the additional libs to be loaded
2315 TString extraLibs;
2316 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2317 // Parse the extra libs for .so
2318 if (fAdditionalLibs.Length()) {
2319 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2320 TIter next(list);
2321 TObjString *str;
2322 while((str=(TObjString*)next()) && str->GetString().Contains(".so")) {
830acc4c 2323 TString stmp = str->GetName();
2324 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2325 stmp.ReplaceAll(".so","");
3bdcb562 2326 if (!extraLibs.IsNull()) extraLibs += ":";
830acc4c 2327 extraLibs += stmp;
3bdcb562 2328 }
2329 if (list) delete list;
2330 }
2331 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2332 // Check extra includes
2333 if (!fIncludePath.IsNull()) {
2334 TString includePath = fIncludePath;
2335 includePath.ReplaceAll(" ",":");
2336 includePath.Strip(TString::kTrailing, ':');
2337 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2338 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2339 }
27734f0e 2340 // Check if connection to grid is requested
2341 if (TestSpecialBit(kProofConnectGrid))
2342 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2343 // Enable AliRoot par
2344 if (testMode) {
2345 // Enable proof lite package
2346 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2347 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2348 TNamed *obj = (TNamed*)optionsList.At(i);
2349 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2350 }
2351 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
2352 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)0x%lx);",alirootLite.Data(),(ULong_t)&optionsList))) {
2353 Info("StartAnalysis", "AliRootProofLite enabled");
2354 } else {
2355 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2356 return kFALSE;
2357 }
2358 } else {
2359 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)0x%lx);",
2360 fAliROOTVersion.Data(), (ULong_t)&optionsList))) {
2361 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2362 return kFALSE;
2363 }
27734f0e 2364 }
3bdcb562 2365 } else {
2366 if (fAdditionalLibs.Contains(".so") && !testMode) {
2367 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2368 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2369 return kFALSE;
2370 }
2371 }
2372 // Enable par files if requested
2373 if (fPackages && fPackages->GetEntries()) {
2374 TIter next(fPackages);
2375 TObject *package;
2376 while ((package=next())) {
2377 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2378 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2379 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2380 return kFALSE;
2381 }
2382 } else {
2383 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2384 return kFALSE;
2385 }
2386 }
2387 }
2388 // Do we need to load analysis source files ?
2389 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2390 if (fAnalysisSource.Length()) {
2391 TObjArray *list = fAnalysisSource.Tokenize(" ");
2392 TIter next(list);
2393 TObjString *str;
2394 while((str=(TObjString*)next())) {
2395 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2396 }
2397 if (list) delete list;
2398 }
830acc4c 2399 if (testMode) {
2400 // Register dataset to proof lite.
2401 if (fFileForTestMode.IsNull()) {
2402 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2403 return kFALSE;
2404 }
2405 if (gSystem->AccessPathName(fFileForTestMode)) {
2406 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2407 return kFALSE;
2408 }
2409 TFileCollection *coll = new TFileCollection();
2410 coll->AddFromFile(fFileForTestMode);
27734f0e 2411 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)0x%lx, \"OV\");", (ULong_t)coll));
2412 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2413 }
3bdcb562 2414 return kTRUE;
2415 }
c57f56b7 2416
43da816a 2417 // Check if output files have to be taken from the analysis manager
2418 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
43da816a 2419 fOutputFiles = "";
2420 TIter next(mgr->GetOutputs());
2421 AliAnalysisDataContainer *output;
2422 while ((output=(AliAnalysisDataContainer*)next())) {
2423 const char *filename = output->GetFileName();
2424 if (!(strcmp(filename, "default"))) {
2425 if (!mgr->GetOutputEventHandler()) continue;
2426 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
2427 }
84fcd93f 2428 if (fOutputFiles.Contains(filename)) continue;
149d288c 2429 if (fOutputFiles.Length()) fOutputFiles += ",";
43da816a 2430 fOutputFiles += filename;
2431 }
c07b9ce2 2432 // Add extra files registered to the analysis manager
2433 if (mgr->GetExtraFiles().Length()) {
149d288c 2434 if (fOutputFiles.Length()) fOutputFiles += ",";
2435 TString extra = mgr->GetExtraFiles();
2436 extra.ReplaceAll(" ", ",");
2437 // Protection in case extra files do not exist (will it work?)
2438 extra.ReplaceAll(".root", "*.root");
2439 fOutputFiles += extra;
84fcd93f 2440 }
149d288c 2441 // Compose the output archive.
2442 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2443 fOutputArchive += Form("root_archive.zip:%s@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2444 }
f7b1cbc2 2445// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2446 if (TestBit(AliAnalysisGrid::kOffline)) {
2447 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2448 \n there nor any job run. You can revise the JDL and analysis \
2449 \n macro then run the same in \"submit\" mode.");
2450 } else if (TestBit(AliAnalysisGrid::kTest)) {
2451 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2452 \n dataset.");
2453 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2454 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2455 \n space and job submitted.");
2456 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2457 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2458 if (fMergeViaJDL) CheckInputData();
5513444a 2459 return kTRUE;
c57f56b7 2460 } else {
2461 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2462 }
2463
348be253 2464 Print();
c57f56b7 2465 if (!Connect()) {
2466 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2467 return kFALSE;
16a4353c 2468 }
d3b18c4c 2469 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2470 if (!CheckInputData()) {
2471 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2472 return kFALSE;
c57f56b7 2473 }
d3339be3 2474 if (!CreateDataset(fDataPattern)) {
2475 TString serror;
2476 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2477 if (fRunNumbers.Length()) serror = "run numbers";
2478 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2479 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2480 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2481 return kFALSE;
2482 }
c57f56b7 2483 WriteAnalysisFile();
2484 WriteAnalysisMacro();
2485 WriteExecutable();
2486 WriteValidationScript();
0f389141 2487 if (fMergeViaJDL) {
2488 WriteMergingMacro();
2489 WriteMergeExecutable();
2490 WriteValidationScript(kTRUE);
2491 }
5513444a 2492 if (!CreateJDL()) return kFALSE;
2493 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2494 if (testMode) {
c57f56b7 2495 // Locally testing the analysis
2496 Info("StartAnalysis", "\n_______________________________________________________________________ \
2497 \n Running analysis script in a daughter shell as on a worker node \
2498 \n_______________________________________________________________________");
149d288c 2499 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2500 TIter next(list);
2501 TObjString *str;
a2f5fc01 2502 TString outputFile;
c57f56b7 2503 while((str=(TObjString*)next())) {
a2f5fc01 2504 outputFile = str->GetString();
2505 Int_t index = outputFile.Index("@");
2506 if (index > 0) outputFile.Remove(index);
2507 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2508 }
2509 delete list;
2510 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 2511 TString validationScript = fExecutable;
2512 validationScript.ReplaceAll(".sh", "_validation.sh");
2513 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 2514// gSystem->Exec("cat stdout");
5513444a 2515 return kFALSE;
c57f56b7 2516 }
5513444a 2517 // Check if submitting is managed by LPM manager
a3e84053 2518 if (fProductionMode) {
5513444a 2519 TString prodfile = fJDLName;
2520 prodfile.ReplaceAll(".jdl", ".prod");
2521 WriteProductionFile(prodfile);
2522 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2523 return kFALSE;
2524 }
a8739e8a 2525 // Submit AliEn job(s)
d2a409b2 2526 gGrid->Cd(fGridOutputDir);
a8739e8a 2527 TGridResult *res;
c57f56b7 2528 TString jobID = "";
d2a409b2 2529 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2530 // Submit a given xml or a set of runs
a8739e8a 2531 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2532 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2533 if (res) {
2534 const char *cjobId = res->GetKey(0,"jobId");
2535 if (!cjobId) {
a03be957 2536 gGrid->Stdout();
2537 gGrid->Stderr();
a8739e8a 2538 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2539 return kFALSE;
a8739e8a 2540 } else {
2541 Info("StartAnalysis", "\n_______________________________________________________________________ \
2542 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2543 \n_______________________________________________________________________",
2544 fJDLName.Data(), cjobId);
2545 jobID = cjobId;
2546 }
2547 delete res;
a03be957 2548 } else {
2549 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2550 return kFALSE;
a8739e8a 2551 }
2552 } else {
d2a409b2 2553 // Submit for a range of enumeration of runs.
a03be957 2554 if (!Submit()) return kFALSE;
c57f56b7 2555 }
a8739e8a 2556
c57f56b7 2557 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2558 \n You may exit at any time and terminate the job later using the option <terminate> \
2559 \n ##################################################################################", jobID.Data());
bb885a9e 2560 gSystem->Exec("aliensh");
5513444a 2561 return kTRUE;
c57f56b7 2562}
2563
d2a409b2 2564//______________________________________________________________________________
a03be957 2565Bool_t AliAnalysisAlien::Submit()
d2a409b2 2566{
2567// Submit all master jobs.
2568 Int_t nmasterjobs = fInputFiles->GetEntries();
2569 Long_t tshoot = gSystem->Now();
a03be957 2570 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2571 while (fNsubmitted < nmasterjobs) {
2572 Long_t now = gSystem->Now();
2573 if ((now-tshoot)>30000) {
2574 tshoot = now;
a03be957 2575 if (!SubmitNext()) return kFALSE;
d2a409b2 2576 }
2577 }
a03be957 2578 return kTRUE;
d2a409b2 2579}
2580
0f389141 2581//______________________________________________________________________________
a03be957 2582Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2583{
2584// Submit all merging jobs.
2585 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2586 gGrid->Cd(fGridOutputDir);
2587 TString mergeJDLName = fExecutable;
2588 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2589 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2590 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2591 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2592 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2593 if (fOutputToRunNo) {
2594 // The output directory is the run number
2595 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2596 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2597 } else {
2598 // The output directory is the master number in 3 digits format
2599 printf("### Submitting merging job for master <%03d>\n", i);
2600 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2601 }
2602 // Check now the number of merging stages.
2b222f7c 2603 TObjArray *list = fOutputFiles.Tokenize(",");
2604 TIter next(list);
2605 TObjString *str;
2606 TString outputFile;
2607 while((str=(TObjString*)next())) {
2608 outputFile = str->GetString();
2609 Int_t index = outputFile.Index("@");
2610 if (index > 0) outputFile.Remove(index);
2611 if (!fMergeExcludes.Contains(outputFile)) break;
2612 }
2613 delete list;
7c2cd90a 2614 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, kTRUE, mergeJDLName);
2615 if (!done) return kFALSE;
0f389141 2616 }
a03be957 2617 if (!ntosubmit) return kTRUE;
0f389141 2618 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
2619 \n You may exit at any time and terminate the job later using the option <terminate> but disabling SetMergeViaJDL\
2620 \n ##################################################################################");
2621 gSystem->Exec("aliensh");
a03be957 2622 return kTRUE;
0f389141 2623}
2624
d2a409b2 2625//______________________________________________________________________________
a03be957 2626Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2627{
08d5b699 2628// Submit next bunch of master jobs if the queue is free. The first master job is
2629// submitted right away, while the next will not be unless the previous was split.
2630// The plugin will not submit new master jobs if there are more that 500 jobs in
2631// waiting phase.
d2a409b2 2632 static Bool_t iscalled = kFALSE;
2633 static Int_t firstmaster = 0;
2634 static Int_t lastmaster = 0;
2635 static Int_t npermaster = 0;
a03be957 2636 if (iscalled) return kTRUE;
d2a409b2 2637 iscalled = kTRUE;
2638 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2639 Int_t ntosubmit = 0;
2640 TGridResult *res;
2641 TString jobID = "";
3bdcb562 2642 Int_t nmasterjobs = fInputFiles->GetEntries();
2643 if (!fNsubmitted) {
2644 ntosubmit = 1;
2645 if (!IsUseSubmitPolicy()) {
90d50a8c 2646 if (nmasterjobs>5)
3bdcb562 2647 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2648 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2649 ntosubmit = nmasterjobs;
2650 }
2651 } else {
d2a409b2 2652 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2653 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2654 // If last master not split, just return
a03be957 2655 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2656 // No more than 100 waiting jobs
08d5b699 2657 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2658 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2659 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2660 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2661 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2662 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2663 }
d2a409b2 2664 for (Int_t i=0; i<ntosubmit; i++) {
2665 // Submit for a range of enumeration of runs.
a03be957 2666 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2667 TString query;
cd11251e 2668 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2669 runOutDir.ReplaceAll(".xml", "");
2670 if (fOutputToRunNo)
2671 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2672 else
2673 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2674 printf("********* %s\n",query.Data());
d2a409b2 2675 res = gGrid->Command(query);
2676 if (res) {
98ca124f 2677 TString cjobId1 = res->GetKey(0,"jobId");
2678 if (!cjobId1.Length()) {
d2a409b2 2679 iscalled = kFALSE;
a03be957 2680 gGrid->Stdout();
2681 gGrid->Stderr();
2682 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2683 return kFALSE;
d2a409b2 2684 } else {
2685 Info("StartAnalysis", "\n_______________________________________________________________________ \
2686 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2687 \n_______________________________________________________________________",
98ca124f 2688 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2689 jobID += cjobId1;
2690 jobID += " ";
98ca124f 2691 lastmaster = cjobId1.Atoi();
d2a409b2 2692 if (!firstmaster) firstmaster = lastmaster;
2693 fNsubmitted++;
2694 }
2695 delete res;
a03be957 2696 } else {
2697 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2698 return kFALSE;
d2a409b2 2699 }
2700 }
2701 iscalled = kFALSE;
a03be957 2702 return kTRUE;
d2a409b2 2703}
2704
c57f56b7 2705//______________________________________________________________________________
2706void AliAnalysisAlien::WriteAnalysisFile()
2707{
f10e8481 2708// Write current analysis manager into the file <analysisFile>
2709 TString analysisFile = fExecutable;
2710 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2711 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2712 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2713 if (!mgr || !mgr->IsInitialized()) {
2714 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2715 return;
2716 }
2717 // Check analysis type
2718 TObject *handler;
2719 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2720 handler = (TObject*)mgr->GetInputEventHandler();
2721 if (handler) {
2722 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2723 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2724 }
2725 TDirectory *cdir = gDirectory;
f10e8481 2726 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2727 if (file) {
e16a394c 2728 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2729 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2730 // Unless merging makes no sense
2731 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2732 mgr->Write();
2733 delete file;
fe2d7fc2 2734 // Enable termination for local jobs
2735 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2736 }
2737 if (cdir) cdir->cd();
f10e8481 2738 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2739 }
2740 Bool_t copy = kTRUE;
d3b18c4c 2741 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 2742 if (copy) {
2743 CdWork();
2744 TString workdir = gGrid->GetHomeDirectory();
2745 workdir += fGridWorkingDir;
d3b18c4c 2746 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
f10e8481 2747 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
2748 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 2749 }
2750}
2751
2752//______________________________________________________________________________
2753void AliAnalysisAlien::WriteAnalysisMacro()
2754{
2755// Write the analysis macro that will steer the analysis in grid mode.
2756 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2757 ofstream out;
2758 out.open(fAnalysisMacro.Data(), ios::out);
2759 if (!out.good()) {
2760 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2761 return;
2762 }
5b273635 2763 Bool_t hasSTEERBase = kFALSE;
2764 Bool_t hasESD = kFALSE;
2765 Bool_t hasAOD = kFALSE;
2766 Bool_t hasANALYSIS = kFALSE;
2767 Bool_t hasANALYSISalice = kFALSE;
2768 Bool_t hasCORRFW = kFALSE;
c57f56b7 2769 TString func = fAnalysisMacro;
2770 TString type = "ESD";
2771 TString comment = "// Analysis using ";
2772 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
2773 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
2774 type = "AOD";
2775 comment += "AOD";
2776 }
0df6ccf2 2777 if (type!="AOD" && fFriendChainName!="") {
2778 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
2779 return;
2780 }
c57f56b7 2781 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
2782 else comment += " data";
2783 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
2784 func.ReplaceAll(".C", "");
2785 out << "void " << func.Data() << "()" << endl;
2786 out << "{" << endl;
2787 out << comment.Data() << endl;
2788 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 2789 out << " TStopwatch timer;" << endl;
2790 out << " timer.Start();" << endl << endl;
13f0c00b 2791 // Change temp directory to current one
2792 out << "// Set temporary merging directory to current one" << endl;
2793 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
c57f56b7 2794 out << "// load base root libraries" << endl;
2795 out << " gSystem->Load(\"libTree\");" << endl;
2796 out << " gSystem->Load(\"libGeom\");" << endl;
2797 out << " gSystem->Load(\"libVMC\");" << endl;
2798 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 2799 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 2800 if (fAdditionalRootLibs.Length()) {
47a4137d 2801 // in principle libtree /lib geom libvmc etc. can go into this list, too
2802 out << "// Add aditional libraries" << endl;
2803 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2804 TIter next(list);
2805 TObjString *str;
2806 while((str=(TObjString*)next())) {
2807 if (str->GetString().Contains(".so"))
2808 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 2809 }
47a4137d 2810 if (list) delete list;
d5c6455a 2811 }
47a4137d 2812 out << "// include path" << endl;
2813 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2814 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
57377eb5 2815 out << "// Load analysis framework libraries" << endl;
652561d5 2816 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 2817 if (!fPackages) {
4e5c5506 2818 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2819 out << " gSystem->Load(\"libESD\");" << endl;
2820 out << " gSystem->Load(\"libAOD\");" << endl;
2821 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 2822 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2823 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 2824 } else {
4e5c5506 2825 TIter next(fPackages);
2826 TObject *obj;
57377eb5 2827 TString pkgname;
57377eb5 2828 while ((obj=next())) {
2829 pkgname = obj->GetName();
4478e6f1 2830 if (pkgname == "STEERBase" ||
2831 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2832 if (pkgname == "ESD" ||
2833 pkgname == "ESD.par") hasESD = kTRUE;
2834 if (pkgname == "AOD" ||
2835 pkgname == "AOD.par") hasAOD = kTRUE;
2836 if (pkgname == "ANALYSIS" ||
2837 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2838 if (pkgname == "ANALYSISalice" ||
2839 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2840 if (pkgname == "CORRFW" ||
2841 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 2842 }
2843 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 2844 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2845 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 2846 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2847 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 2848 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2849 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 2850 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2851 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 2852 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2853 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 2854 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2855 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 2856 out << "// Compile other par packages" << endl;
2857 next.Reset();
fcc9bb6f 2858 while ((obj=next())) {
2859 pkgname = obj->GetName();
4478e6f1 2860 if (pkgname == "STEERBase" ||
2861 pkgname == "STEERBase.par" ||
2862 pkgname == "ESD" ||
2863 pkgname == "ESD.par" ||
2864 pkgname == "AOD" ||
2865 pkgname == "AOD.par" ||
2866 pkgname == "ANALYSIS" ||
2867 pkgname == "ANALYSIS.par" ||
2868 pkgname == "ANALYSISalice" ||
2869 pkgname == "ANALYSISalice.par" ||
2870 pkgname == "CORRFW" ||
2871 pkgname == "CORRFW.par") continue;
5b273635 2872 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 2873 }
4e5c5506 2874 }
6da75e0b 2875 if (fAdditionalLibs.Length()) {
2876 out << "// Add aditional AliRoot libraries" << endl;
2877 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2878 TIter next(list);
2879 TObjString *str;
2880 while((str=(TObjString*)next())) {
2881 if (str->GetString().Contains(".so"))
2882 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 2883 if (str->GetString().Contains(".par"))
2884 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 2885 }
2886 if (list) delete list;
2887 }
2888 out << endl;
c57f56b7 2889 out << "// analysis source to be compiled at runtime (if any)" << endl;
2890 if (fAnalysisSource.Length()) {
2891 TObjArray *list = fAnalysisSource.Tokenize(" ");
2892 TIter next(list);
2893 TObjString *str;
2894 while((str=(TObjString*)next())) {
2895 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2896 }
2897 if (list) delete list;
2898 }
2899 out << endl;
0f389141 2900 if (fFastReadOption) {
d3339be3 2901 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
2902 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 2903 out << "// fast xrootd reading enabled" << endl;
2904 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2905 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
9a8288ff 2906 out << " gEnv->SetValue(\"XNet.RequestTimeout\",20);" << endl;
0f389141 2907 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
9a8288ff 2908 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 2909 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2910 }
c57f56b7 2911 out << "// connect to AliEn and make the chain" << endl;
2912 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2913 if (IsUsingTags()) {
2914 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
2915 } else {
45e980c4 2916 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
c57f56b7 2917 }
2918 out << "// read the analysis manager from file" << endl;
f10e8481 2919 TString analysisFile = fExecutable;
2920 analysisFile.ReplaceAll(".sh", ".root");
2921 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 2922 out << " if (!file) return;" << endl;
2923 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2924 out << " AliAnalysisManager *mgr = 0;" << endl;
2925 out << " TKey *key;" << endl;
2926 out << " while ((key=(TKey*)nextkey())) {" << endl;
2927 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2928 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2929 out << " };" << endl;
2930 out << " if (!mgr) {" << endl;
205b201f 2931 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 2932 out << " return;" << endl;
2933 out << " }" << endl << endl;
2934 out << " mgr->PrintStatus();" << endl;
52b6a92b 2935 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 2936 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 2937 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 2938 } else {
9a8288ff 2939 if (TestBit(AliAnalysisGrid::kTest))
2940 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
2941 else
2942 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 2943 }
2944 }
c57f56b7 2945 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 2946 out << " timer.Stop();" << endl;
2947 out << " timer.Print();" << endl;
c57f56b7 2948 out << "}" << endl << endl;
2949 if (IsUsingTags()) {
2950 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
2951 out << "{" << endl;
2952 out << "// Create a chain using tags from the xml file." << endl;
2953 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
2954 out << " if (!coll) {" << endl;
2955 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2956 out << " return NULL;" << endl;
2957 out << " }" << endl;
2958 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
2959 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
2960 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
2961 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
2962 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
2963 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
2964 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
2965 out << " // Check if the cuts configuration file was provided" << endl;
2966 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
2967 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
2968 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2969 out << " }" << endl;
0df6ccf2 2970 if (fFriendChainName=="") {
2971 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2972 } else {
2973 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
2974 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
2975 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
2976 }
c57f56b7 2977 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
2978 out << " chain->ls();" << endl;
2979 out << " return chain;" << endl;
fcc9bb6f 2980 out << "}" << endl << endl;
c57f56b7 2981 if (gSystem->AccessPathName("ConfigureCuts.C")) {
2982 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
2983 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
2984 msg += " AliLHCTagCuts *lhcCuts,\n";
2985 msg += " AliDetectorTagCuts *detCuts,\n";
2986 msg += " AliEventTagCuts *evCuts)";
23329835 2987 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 2988 }
0df6ccf2 2989 }
2990 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 2991 out <<"//________________________________________________________________________________" << endl;
c57f56b7 2992 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
2993 out << "{" << endl;
2994 out << "// Create a chain using url's from xml file" << endl;
2995 out << " TString treename = type;" << endl;
2996 out << " treename.ToLower();" << endl;
2997 out << " treename += \"Tree\";" << endl;
e02fee64 2998 out << " printf(\"***************************************\\n\");" << endl;
2999 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3000 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 3001 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3002 out << " if (!coll) {" << endl;
3003 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3004 out << " return NULL;" << endl;
3005 out << " }" << endl;
3006 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 3007 if(fFriendChainName!="") {
3008 out << " TChain *chainFriend = new TChain(treename);" << endl;
3009 }
c57f56b7 3010 out << " coll->Reset();" << endl;
0df6ccf2 3011 out << " while (coll->Next()) {" << endl;
3012 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
3013 if(fFriendChainName!="") {
3014 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3015 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3016 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3017 out << " chainFriend->Add(fileFriend.Data());" << endl;
3018 }
3019 out << " }" << endl;
c57f56b7 3020 out << " if (!chain->GetNtrees()) {" << endl;
3021 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3022 out << " return NULL;" << endl;
3023 out << " }" << endl;
0df6ccf2 3024 if(fFriendChainName!="") {
3025 out << " chain->AddFriend(chainFriend);" << endl;
3026 }
c57f56b7 3027 out << " return chain;" << endl;
fcc9bb6f 3028 out << "}" << endl << endl;
c57f56b7 3029 }
5b273635 3030 if (hasANALYSISalice) {
3031 out <<"//________________________________________________________________________________" << endl;
3032 out << "Bool_t SetupPar(const char *package) {" << endl;
3033 out << "// Compile the package and set it up." << endl;
3034 out << " TString pkgdir = package;" << endl;
3035 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
3036 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
3037 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3038 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3039 out << " // Check for BUILD.sh and execute" << endl;
3040 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3041 out << " printf(\"*******************************\\n\");" << endl;
3042 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3043 out << " printf(\"*******************************\\n\");" << endl;
3044 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3045 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3046 out << " gSystem->ChangeDirectory(cdir);" << endl;
3047 out << " return kFALSE;" << endl;
3048 out << " }" << endl;
3049 out << " } else {" << endl;
3050 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3051 out << " gSystem->ChangeDirectory(cdir);" << endl;
3052 out << " return kFALSE;" << endl;
3053 out << " }" << endl;
3054 out << " // Check for SETUP.C and execute" << endl;
3055 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3056 out << " printf(\"*******************************\\n\");" << endl;
3057 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3058 out << " printf(\"*******************************\\n\");" << endl;
3059 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3060 out << " } else {" << endl;
3061 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3062 out << " gSystem->ChangeDirectory(cdir);" << endl;
3063 out << " return kFALSE;" << endl;
3064 out << " }" << endl;
3065 out << " // Restore original workdir" << endl;
3066 out << " gSystem->ChangeDirectory(cdir);" << endl;
3067 out << " return kTRUE;" << endl;
3068 out << "}" << endl;
3069 }
c57f56b7 3070 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3071 }
3072 Bool_t copy = kTRUE;
d3b18c4c 3073 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3074 if (copy) {
3075 CdWork();
3076 TString workdir = gGrid->GetHomeDirectory();
3077 workdir += fGridWorkingDir;
3078 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3079 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3080 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3081 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3082 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3083 }
3084 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3085 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3086 }
3087}
3088
0f389141 3089//______________________________________________________________________________
3090void AliAnalysisAlien::WriteMergingMacro()
3091{
3092// Write a macro to merge the outputs per master job.
3093 if (!fMergeViaJDL) return;
3094 if (!fOutputFiles.Length()) {
3095 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3096 return;
3097 }
3098 TString mergingMacro = fExecutable;
3099 mergingMacro.ReplaceAll(".sh","_merge.C");
3100 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3101 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3102 ofstream out;
3103 out.open(mergingMacro.Data(), ios::out);
3104 if (!out.good()) {
3105 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3106 return;
3107 }
5b273635 3108 Bool_t hasSTEERBase = kFALSE;
3109 Bool_t hasESD = kFALSE;
3110 Bool_t hasAOD = kFALSE;
3111 Bool_t hasANALYSIS = kFALSE;
3112 Bool_t hasANALYSISalice = kFALSE;
3113 Bool_t hasCORRFW = kFALSE;
0f389141 3114 TString func = mergingMacro;
3115 TString comment;
3116 func.ReplaceAll(".C", "");
7c2cd90a 3117 out << "void " << func.Data() << "(const char *dir, Int_t stage=0, Int_t ichunk=0)" << endl;
0f389141 3118 out << "{" << endl;
3119 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3120 out << " TStopwatch timer;" << endl;
3121 out << " timer.Start();" << endl << endl;
7c2cd90a 3122 if (!fExecutableCommand.Contains("aliroot")) {
3123 out << "// load base root libraries" << endl;
3124 out << " gSystem->Load(\"libTree\");" << endl;
3125 out << " gSystem->Load(\"libGeom\");" << endl;
3126 out << " gSystem->Load(\"libVMC\");" << endl;
3127 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3128 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3129 }
0f389141 3130 if (fAdditionalRootLibs.Length()) {
3131 // in principle libtree /lib geom libvmc etc. can go into this list, too
3132 out << "// Add aditional libraries" << endl;
3133 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3134 TIter next(list);
3135 TObjString *str;
3136 while((str=(TObjString*)next())) {
3137 if (str->GetString().Contains(".so"))
3138 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3139 }
3140 if (list) delete list;
3141 }
3142 out << "// include path" << endl;
3143 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3144 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
3145 out << "// Load analysis framework libraries" << endl;
3146 if (!fPackages) {
7c2cd90a 3147 if (!fExecutableCommand.Contains("aliroot")) {
3148 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3149 out << " gSystem->Load(\"libESD\");" << endl;
3150 out << " gSystem->Load(\"libAOD\");" << endl;
3151 }
0f389141 3152 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3153 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3154 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3155 } else {
3156 TIter next(fPackages);
3157 TObject *obj;
3158 TString pkgname;
5b273635 3159 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3160 while ((obj=next())) {
3161 pkgname = obj->GetName();
3162 if (pkgname == "STEERBase" ||
3163 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3164 if (pkgname == "ESD" ||
3165 pkgname == "ESD.par") hasESD = kTRUE;
3166 if (pkgname == "AOD" ||
3167 pkgname == "AOD.par") hasAOD = kTRUE;
3168 if (pkgname == "ANALYSIS" ||
3169 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3170 if (pkgname == "ANALYSISalice" ||
3171 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3172 if (pkgname == "CORRFW" ||
3173 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3174 }
5b273635 3175 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3176 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3177 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3178 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3179 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3180 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3181 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3182 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3183 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3184 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3185 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3186 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3187 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3188 out << "// Compile other par packages" << endl;
3189 next.Reset();
3190 while ((obj=next())) {
3191 pkgname = obj->GetName();
3192 if (pkgname == "STEERBase" ||
3193 pkgname == "STEERBase.par" ||
3194 pkgname == "ESD" ||
3195 pkgname == "ESD.par" ||
3196 pkgname == "AOD" ||
3197 pkgname == "AOD.par" ||
3198 pkgname == "ANALYSIS" ||
3199 pkgname == "ANALYSIS.par" ||
3200 pkgname == "ANALYSISalice" ||
3201 pkgname == "ANALYSISalice.par" ||
3202 pkgname == "CORRFW" ||
3203 pkgname == "CORRFW.par") continue;
5b273635 3204 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3205 }
3206 }
3207 if (fAdditionalLibs.Length()) {
3208 out << "// Add aditional AliRoot libraries" << endl;
3209 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3210 TIter next(list);
3211 TObjString *str;
3212 while((str=(TObjString*)next())) {
3213 if (str->GetString().Contains(".so"))
3214 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3215 }
3216 if (list) delete list;
3217 }
3218 out << endl;
3219 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3220 if (fAnalysisSource.Length()) {
3221 TObjArray *list = fAnalysisSource.Tokenize(" ");
3222 TIter next(list);
3223 TObjString *str;
3224 while((str=(TObjString*)next())) {
3225 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3226 }
3227 if (list) delete list;
3228 }
149d288c 3229 out << endl;
3230
0f389141 3231 if (fFastReadOption) {
3232 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3233 out << "// fast xrootd reading enabled" << endl;
3234 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 3235 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
9a8288ff 3236 out << " gEnv->SetValue(\"XNet.RequestTimeout\",20);" << endl;
0f389141 3237 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
9a8288ff 3238 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
0f389141 3239 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3240 }
3241 // Change temp directory to current one
3242 out << "// Set temporary merging directory to current one" << endl;
3243 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3244 out << "// Connect to AliEn" << endl;
3245 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0cdf65a8 3246 out << " Bool_t laststage = kFALSE;" << endl;
7c2cd90a 3247 out << " TString outputDir = dir;" << endl;
0f389141 3248 out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
3249 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
1f0d1ca2 3250 out << " mergeExcludes += \"" << AliAnalysisManager::GetAnalysisManager()->GetExtraFiles() << "\";" << endl;
58268c13 3251 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3252 out << " TIter *iter = new TIter(list);" << endl;
3253 out << " TObjString *str;" << endl;
a2f5fc01 3254 out << " TString outputFile;" << endl;
0f389141 3255 out << " Bool_t merged = kTRUE;" << endl;
3256 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3257 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3258 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3259 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3260 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3261 out << " // Skip already merged outputs" << endl;
a2f5fc01 3262 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3263 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3264 out << " continue;" << endl;
3265 out << " }" << endl;
a2f5fc01 3266 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
7c2cd90a 3267 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage, ichunk);" << endl;
0f389141 3268 out << " if (!merged) {" << endl;
a2f5fc01 3269 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3270 out << " return;" << endl;
0f389141 3271 out << " }" << endl;
7c2cd90a 3272 out << " // Check if this was the last stage. If yes, run terminate for the tasks." << endl;
3273 out << " if (!gSystem->AccessPathName(outputFile)) laststage = kTRUE;" << endl;
0f389141 3274 out << " }" << endl;
7c2cd90a 3275 out << " // all outputs merged, validate" << endl;
3276 out << " ofstream out;" << endl;
3277 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3278 out << " out.close();" << endl;
3279 out << " // read the analysis manager from file" << endl;
f866cba5 3280 TString analysisFile = fExecutable;
3281 analysisFile.ReplaceAll(".sh", ".root");
7c2cd90a 3282 out << " if (!laststage) return;" << endl;
f866cba5 3283 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3284 out << " if (!file) return;" << endl;
f866cba5 3285 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3286 out << " AliAnalysisManager *mgr = 0;" << endl;
3287 out << " TKey *key;" << endl;
3288 out << " while ((key=(TKey*)nextkey())) {" << endl;
3289 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3290 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3291 out << " };" << endl;
3292 out << " if (!mgr) {" << endl;
3293 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3294 out << " return;" << endl;
3295 out << " }" << endl << endl;
b385fec0 3296 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3297 out << " mgr->PrintStatus();" << endl;
3298 if (AliAnalysisManager::GetAnalysisManager()) {
3299 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3300 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3301 } else {
9a8288ff 3302 if (TestBit(AliAnalysisGrid::kTest))
3303 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3304 else
3305 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
f866cba5 3306 }
3307 }
9a8288ff 3308 out << " TTree *tree = NULL;" << endl;
3309 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
0f389141 3310 out << "}" << endl << endl;
5b273635 3311 if (hasANALYSISalice) {
3312 out <<"//________________________________________________________________________________" << endl;
3313 out << "Bool_t SetupPar(const char *package) {" << endl;
3314 out << "// Compile the package and set it up." << endl;
3315 out << " TString pkgdir = package;" << endl;
3316 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
3317 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
3318 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3319 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3320 out << " // Check for BUILD.sh and execute" << endl;
3321 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3322 out << " printf(\"*******************************\\n\");" << endl;
3323 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3324 out << " printf(\"*******************************\\n\");" << endl;
3325 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3326 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3327 out << " gSystem->ChangeDirectory(cdir);" << endl;
3328 out << " return kFALSE;" << endl;
3329 out << " }" << endl;
3330 out << " } else {" << endl;
3331 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3332 out << " gSystem->ChangeDirectory(cdir);" << endl;
3333 out << " return kFALSE;" << endl;
3334 out << " }" << endl;
3335 out << " // Check for SETUP.C and execute" << endl;
3336 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3337 out << " printf(\"*******************************\\n\");" << endl;
3338 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3339 out << " printf(\"*******************************\\n\");" << endl;
3340 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3341 out << " } else {" << endl;
3342 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3343 out << " gSystem->ChangeDirectory(cdir);" << endl;
3344 out << " return kFALSE;" << endl;
3345 out << " }" << endl;
3346 out << " // Restore original workdir" << endl;
3347 out << " gSystem->ChangeDirectory(cdir);" << endl;
3348 out << " return kTRUE;" << endl;
3349 out << "}" << endl;
3350 }
0f389141 3351 }
3352 Bool_t copy = kTRUE;
d3b18c4c 3353 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3354 if (copy) {
3355 CdWork();
3356 TString workdir = gGrid->GetHomeDirectory();
3357 workdir += fGridWorkingDir;
3358 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3359 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3360 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3361 }
3362}
3363
3364//______________________________________________________________________________
3365Bool_t AliAnalysisAlien::SetupPar(const char *package)
3366{
205b201f 3367// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3368// Note that for loading the compiled library. The current directory should have precedence in
3369// LD_LIBRARY_PATH
3370 TString pkgdir = package;
3371 pkgdir.ReplaceAll(".par","");
3372 gSystem->Exec(Form("tar xvzf %s.par", pkgdir.Data()));
3373 TString cdir = gSystem->WorkingDirectory();
3374 gSystem->ChangeDirectory(pkgdir);
3375 // Check for BUILD.sh and execute
3376 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3377 printf("**************************************************\n");
3378 printf("*** Building PAR archive %s\n", package);
3379 printf("**************************************************\n");
3380 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3381 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3382 gSystem->ChangeDirectory(cdir);
3383 return kFALSE;
3384 }
3385 } else {
3386 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3387 gSystem->ChangeDirectory(cdir);
3388 return kFALSE;
3389 }
3390 // Check for SETUP.C and execute
3391 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3392 printf("**************************************************\n");
3393 printf("*** Setup PAR archive %s\n", package);
3394 printf("**************************************************\n");
3395 gROOT->Macro("PROOF-INF/SETUP.C");
3396 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3397 } else {
3398 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3399 gSystem->ChangeDirectory(cdir);
3400 return kFALSE;
3401 }
3402 // Restore original workdir
3403 gSystem->ChangeDirectory(cdir);
3404 return kTRUE;
3405}
3406
c57f56b7 3407//______________________________________________________________________________
3408void AliAnalysisAlien::WriteExecutable()
3409{
3410// Generate the alien executable script.
3411 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3412 ofstream out;
3413 out.open(fExecutable.Data(), ios::out);
3414 if (out.bad()) {
5513444a 3415 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3416 return;
3417 }
3418 out << "#!/bin/bash" << endl;
c57f56b7 3419 out << "echo \"=========================================\"" << endl;
3420 out << "echo \"############## PATH : ##############\"" << endl;
3421 out << "echo $PATH" << endl;
3422 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3423 out << "echo $LD_LIBRARY_PATH" << endl;
3424 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3425 out << "echo $ROOTSYS" << endl;
3426 out << "echo \"############## which root : ##############\"" << endl;
3427 out << "which root" << endl;
3428 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3429 out << "echo $ALICE_ROOT" << endl;
3430 out << "echo \"############## which aliroot : ##############\"" << endl;
3431 out << "which aliroot" << endl;
9c5ddadc 3432 out << "echo \"############## system limits : ##############\"" << endl;
3433 out << "ulimit -a" << endl;
3434 out << "echo \"############## memory : ##############\"" << endl;
3435 out << "free -m" << endl;
c57f56b7 3436 out << "echo \"=========================================\"" << endl << endl;
74a53467 3437 // Make sure we can properly compile par files
3438 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 3439 out << fExecutableCommand << " ";
631c0b05 3440 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3441 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3442 out << "echo \"############## memory after: ##############\"" << endl;
3443 out << "free -m" << endl;
c57f56b7 3444 }
3445 Bool_t copy = kTRUE;
d3b18c4c 3446 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3447 if (copy) {
3448 CdWork();
3449 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3450 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3451 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3452 workdir += fGridWorkingDir;
3453 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3454 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3455 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
c57f56b7 3456 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3457 }
3458}
3459
0f389141 3460//______________________________________________________________________________
3461void AliAnalysisAlien::WriteMergeExecutable()
3462{
3463// Generate the alien executable script for the merging job.
3464 if (!fMergeViaJDL) return;
3465 TString mergeExec = fExecutable;
3466 mergeExec.ReplaceAll(".sh", "_merge.sh");
3467 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3468 ofstream out;
3469 out.open(mergeExec.Data(), ios::out);
3470 if (out.bad()) {
3471 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3472 return;
3473 }
3474 out << "#!/bin/bash" << endl;
3475 out << "echo \"=========================================\"" << endl;
3476 out << "echo \"############## PATH : ##############\"" << endl;
3477 out << "echo $PATH" << endl;
3478 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3479 out << "echo $LD_LIBRARY_PATH" << endl;
3480 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3481 out << "echo $ROOTSYS" << endl;
3482 out << "echo \"############## which root : ##############\"" << endl;
3483 out << "which root" << endl;
3484 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3485 out << "echo $ALICE_ROOT" << endl;
3486 out << "echo \"############## which aliroot : ##############\"" << endl;
3487 out << "which aliroot" << endl;
3488 out << "echo \"############## system limits : ##############\"" << endl;
3489 out << "ulimit -a" << endl;
3490 out << "echo \"############## memory : ##############\"" << endl;
3491 out << "free -m" << endl;
3492 out << "echo \"=========================================\"" << endl << endl;
3493 // Make sure we can properly compile par files
3494 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
3495 TString mergeMacro = fExecutable;
3496 mergeMacro.ReplaceAll(".sh", "_merge.C");
7c2cd90a 3497 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2,$3)\"" << endl;
0f389141 3498 out << fExecutableCommand << " " << "$ARG" << endl;
3499 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3500 out << "echo \"############## memory after: ##############\"" << endl;
3501 out << "free -m" << endl;
0f389141 3502 }
3503 Bool_t copy = kTRUE;
d3b18c4c 3504 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
0f389141 3505 if (copy) {
3506 CdWork();
3507 TString workdir = gGrid->GetHomeDirectory();
3508 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3509 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3510 workdir += fGridWorkingDir;
3511 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3512 if (FileExists(executable)) gGrid->Rm(executable);
d3b18c4c 3513 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
0f389141 3514 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3515 }
3516}
3517
c57f56b7 3518//______________________________________________________________________________
5513444a 3519void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3520{
3521// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3522// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3523// Next lines: full_path_to_dataset XXX (XXX is a string)
3524// To submit, one has to: submit jdl XXX for all lines
3525 ofstream out;
3526 out.open(filename, ios::out);
3527 if (out.bad()) {
3528 Error("WriteProductionFile", "Bad file name: %s", filename);
3529 return;
3530 }
d3b18c4c 3531 TString workdir;
3532 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3533 workdir = gGrid->GetHomeDirectory();
5513444a 3534 workdir += fGridWorkingDir;
f5e8c702 3535 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3536 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3537 out << locjdl << " " << njobspermaster << endl;
5513444a 3538 Int_t nmasterjobs = fInputFiles->GetEntries();
3539 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3540 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3541 runOutDir.ReplaceAll(".xml", "");
3542 if (fOutputToRunNo)
3543 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3544 else
3545 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3546 }
d3b18c4c 3547 if (gGrid) {
3548 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3549 if (FileExists(filename)) gGrid->Rm(filename);
3550 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3551 }
5513444a 3552}
3553
3554//______________________________________________________________________________
0f389141 3555void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3556{
3557// Generate the alien validation script.
3558 // Generate the validation script
3559 TObjString *os;
d3b18c4c 3560 if (fValidationScript.IsNull()) {
3561 fValidationScript = fExecutable;
3562 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3563 }
3564 TString validationScript = fValidationScript;
3565 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
c57f56b7 3566 if (!Connect()) {
3567 Error("WriteValidationScript", "Alien connection required");
3568 return;
3569 }
d3b18c4c 3570 if (!fTerminateFiles.IsNull()) {
3571 fTerminateFiles.Strip();
3572 fTerminateFiles.ReplaceAll(" ",",");
3573 }
a2f5fc01 3574 TString outStream = "";
3575 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3576 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3577 ofstream out;
0d5d317c 3578 out.open(validationScript, ios::out);
c57f56b7 3579 out << "#!/bin/bash" << endl;
3580 out << "##################################################" << endl;
3581 out << "validateout=`dirname $0`" << endl;
3582 out << "validatetime=`date`" << endl;
3583 out << "validated=\"0\";" << endl;
3584 out << "error=0" << endl;
3585 out << "if [ -z $validateout ]" << endl;
3586 out << "then" << endl;
3587 out << " validateout=\".\"" << endl;
3588 out << "fi" << endl << endl;
3589 out << "cd $validateout;" << endl;
3590 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3591 out << "echo \"*******************************************************\"" << outStream << endl;
3592 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3593 out << "" << endl;
a2f5fc01 3594 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3595 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3596 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3597 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3598 out << "ls -la ./" << outStream << endl;
3599 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3600 out << "##################################################" << endl;
ebec370a 3601 out << "" << endl;
3602
3603 out << "if [ ! -f stderr ] ; then" << endl;
3604 out << " error=1" << endl;
a2f5fc01 3605 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3606 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3607 out << "fi" << endl;
3608
b34c9f51 3609 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3610 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3611 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3612 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3613 out << "" << endl;
3614
ebec370a 3615 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3616 out << " error=1" << endl;
a2f5fc01 3617 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3618 out << " echo \"$parArch\" " << outStream << endl;
3619 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3620 out << "fi" << endl;
3621
3622 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3623 out << " error=1" << endl;
a2f5fc01 3624 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3625 out << " echo \"$segViol\" " << outStream << endl;
3626 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3627 out << "fi" << endl;
3628
3629 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3630 out << " error=1" << endl;
a2f5fc01 3631 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3632 out << " echo \"$segFault\" " << outStream << endl;
3633 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3634 out << "fi" << endl;
3635
b34c9f51 3636 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3637 out << " error=1" << endl;
a2f5fc01 3638 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3639 out << " echo \"$glibcErr\" " << outStream << endl;
3640 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3641 out << "fi" << endl;
3642
ebec370a 3643 // Part dedicated to the specific analyses running into the train
3644
d3b18c4c 3645 TString outputFiles = fOutputFiles;
3646 if (merge && !fTerminateFiles.IsNull()) {
3647 outputFiles += ",";
3648 outputFiles += fTerminateFiles;
3649 }
3650 TObjArray *arr = outputFiles.Tokenize(",");
c57f56b7 3651 TIter next1(arr);
a2f5fc01 3652 TString outputFile;
5b9b4998 3653 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3654 TString extra = mgr->GetExtraFiles();
c57f56b7 3655 while ((os=(TObjString*)next1())) {
a2f5fc01 3656 outputFile = os->GetString();
3657 Int_t index = outputFile.Index("@");
3658 if (index > 0) outputFile.Remove(index);
d3b18c4c 3659 if (!merge && fTerminateFiles.Contains(outputFile)) continue;
a2f5fc01 3660 if (merge && fMergeExcludes.Contains(outputFile)) continue;
3661 if (extra.Contains(outputFile)) continue;
3662 if (outputFile.Contains("*")) continue;
3663 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3664 out << " error=1" << endl;
7c2cd90a 3665 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3666 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3667 out << "fi" << endl;
3668 }
3669 delete arr;
7c2cd90a 3670 out << "if ! [ -f outputs_valid ] ; then" << endl;
3671 out << " error=1" << endl;
3672 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3673 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3674 out << "fi" << endl;
923e2ca5 3675
c57f56b7 3676 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3677 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3678 if (!IsKeepLogs()) {
3679 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3680 outStream = "";
149d288c 3681 out << " rm -f std*" << endl;
3682 }
c57f56b7 3683 out << "fi" << endl;
3684
a2f5fc01 3685 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3686 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3687 out << "cd -" << endl;
3688 out << "exit $error" << endl;
3689 }
3690 Bool_t copy = kTRUE;
d3b18c4c 3691 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
c57f56b7 3692 if (copy) {
3693 CdWork();
3694 TString workdir = gGrid->GetHomeDirectory();
3695 workdir += fGridWorkingDir;
d3b18c4c 3696 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
0d5d317c 3697 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3698 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3699 }
3700}