]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
remove left over print statement
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
7c2cd90a 25#include "TBits.h"
0f389141 26#include "TError.h"
c57f56b7 27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
830acc4c 30#include "TFileCollection.h"
3bdcb562 31#include "TChain.h"
c57f56b7 32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
d2a409b2 38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
c57f56b7 40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
bb885a9e 42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
c57f56b7 44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
0f389141 52 fMergingJDL(NULL),
c57f56b7 53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
bb885a9e 58 fNtestFiles(0),
319593fb 59 fNrunsPerMaster(0),
16a4353c 60 fMaxMergeFiles(0),
d2a409b2 61 fNsubmitted(0),
a3e84053 62 fProductionMode(0),
cd11251e 63 fOutputToRunNo(0),
0f389141 64 fMergeViaJDL(0),
65 fFastReadOption(0),
e1c22e21 66 fOverwriteMode(1),
149d288c 67 fNreplicas(2),
3bdcb562 68 fNproofWorkers(0),
27734f0e 69 fNproofWorkersPerSlave(0),
3bdcb562 70 fProofReset(0),
c57f56b7 71 fRunNumbers(),
72 fExecutable(),
0a1c1f7f 73 fExecutableCommand(),
c57f56b7 74 fArguments(),
631c0b05 75 fExecutableArgs(),
c57f56b7 76 fAnalysisMacro(),
77 fAnalysisSource(),
d5c6455a 78 fAdditionalRootLibs(),
c57f56b7 79 fAdditionalLibs(),
80 fSplitMode(),
81 fAPIVersion(),
82 fROOTVersion(),
83 fAliROOTVersion(),
648174cf 84 fExternalPackages(),
c57f56b7 85 fUser(),
86 fGridWorkingDir(),
87 fGridDataDir(),
88 fDataPattern(),
89 fGridOutputDir(),
90 fOutputArchive(),
91 fOutputFiles(),
92 fInputFormat(),
e7c71df0 93 fDatasetName(),
c57f56b7 94 fJDLName(),
bb885a9e 95 fMergeExcludes(),
f965131e 96 fIncludePath(),
bb885a9e 97 fCloseSE(),
0df6ccf2 98 fFriendChainName(),
c6cb3634 99 fJobTag(),
648174cf 100 fOutputSingle(),
5fce53f4 101 fRunPrefix(),
3bdcb562 102 fProofCluster(),
103 fProofDataSet(),
104 fFileForTestMode(),
105 fRootVersionForProof(),
106 fAliRootMode(),
4e5c5506 107 fInputFiles(0),
108 fPackages(0)
c57f56b7 109{
110// Dummy ctor.
111 SetDefaults();
112}
113
114//______________________________________________________________________________
115AliAnalysisAlien::AliAnalysisAlien(const char *name)
116 :AliAnalysisGrid(name),
117 fGridJDL(NULL),
0f389141 118 fMergingJDL(NULL),
c57f56b7 119 fPrice(0),
120 fTTL(0),
121 fSplitMaxInputFileNumber(0),
122 fMaxInitFailed(0),
123 fMasterResubmitThreshold(0),
bb885a9e 124 fNtestFiles(0),
319593fb 125 fNrunsPerMaster(0),
16a4353c 126 fMaxMergeFiles(0),
d2a409b2 127 fNsubmitted(0),
a3e84053 128 fProductionMode(0),
cd11251e 129 fOutputToRunNo(0),
0f389141 130 fMergeViaJDL(0),
131 fFastReadOption(0),
e1c22e21 132 fOverwriteMode(1),
149d288c 133 fNreplicas(2),
3bdcb562 134 fNproofWorkers(0),
27734f0e 135 fNproofWorkersPerSlave(0),
3bdcb562 136 fProofReset(0),
c57f56b7 137 fRunNumbers(),
138 fExecutable(),
0a1c1f7f 139 fExecutableCommand(),
c57f56b7 140 fArguments(),
631c0b05 141 fExecutableArgs(),
c57f56b7 142 fAnalysisMacro(),
143 fAnalysisSource(),
d5c6455a 144 fAdditionalRootLibs(),
c57f56b7 145 fAdditionalLibs(),
146 fSplitMode(),
147 fAPIVersion(),
148 fROOTVersion(),
149 fAliROOTVersion(),
648174cf 150 fExternalPackages(),
c57f56b7 151 fUser(),
152 fGridWorkingDir(),
153 fGridDataDir(),
154 fDataPattern(),
155 fGridOutputDir(),
156 fOutputArchive(),
157 fOutputFiles(),
158 fInputFormat(),
e7c71df0 159 fDatasetName(),
c57f56b7 160 fJDLName(),
bb885a9e 161 fMergeExcludes(),
f965131e 162 fIncludePath(),
bb885a9e 163 fCloseSE(),
0df6ccf2 164 fFriendChainName(),
c6cb3634 165 fJobTag(),
648174cf 166 fOutputSingle(),
5fce53f4 167 fRunPrefix(),
3bdcb562 168 fProofCluster(),
169 fProofDataSet(),
170 fFileForTestMode(),
171 fRootVersionForProof(),
172 fAliRootMode(),
4e5c5506 173 fInputFiles(0),
174 fPackages(0)
c57f56b7 175{
176// Default ctor.
177 SetDefaults();
178}
179
180//______________________________________________________________________________
181AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
182 :AliAnalysisGrid(other),
183 fGridJDL(NULL),
0f389141 184 fMergingJDL(NULL),
c57f56b7 185 fPrice(other.fPrice),
186 fTTL(other.fTTL),
187 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
188 fMaxInitFailed(other.fMaxInitFailed),
189 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 190 fNtestFiles(other.fNtestFiles),
319593fb 191 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 192 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 193 fNsubmitted(other.fNsubmitted),
a3e84053 194 fProductionMode(other.fProductionMode),
cd11251e 195 fOutputToRunNo(other.fOutputToRunNo),
0f389141 196 fMergeViaJDL(other.fMergeViaJDL),
197 fFastReadOption(other.fFastReadOption),
198 fOverwriteMode(other.fOverwriteMode),
149d288c 199 fNreplicas(other.fNreplicas),
3bdcb562 200 fNproofWorkers(other.fNproofWorkers),
27734f0e 201 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
3bdcb562 202 fProofReset(other.fProofReset),
c57f56b7 203 fRunNumbers(other.fRunNumbers),
204 fExecutable(other.fExecutable),
0a1c1f7f 205 fExecutableCommand(other.fExecutableCommand),
c57f56b7 206 fArguments(other.fArguments),
631c0b05 207 fExecutableArgs(other.fExecutableArgs),
c57f56b7 208 fAnalysisMacro(other.fAnalysisMacro),
209 fAnalysisSource(other.fAnalysisSource),
d5c6455a 210 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 211 fAdditionalLibs(other.fAdditionalLibs),
212 fSplitMode(other.fSplitMode),
213 fAPIVersion(other.fAPIVersion),
214 fROOTVersion(other.fROOTVersion),
215 fAliROOTVersion(other.fAliROOTVersion),
648174cf 216 fExternalPackages(other.fExternalPackages),
c57f56b7 217 fUser(other.fUser),
218 fGridWorkingDir(other.fGridWorkingDir),
219 fGridDataDir(other.fGridDataDir),
220 fDataPattern(other.fDataPattern),
221 fGridOutputDir(other.fGridOutputDir),
222 fOutputArchive(other.fOutputArchive),
223 fOutputFiles(other.fOutputFiles),
224 fInputFormat(other.fInputFormat),
e7c71df0 225 fDatasetName(other.fDatasetName),
c57f56b7 226 fJDLName(other.fJDLName),
bb885a9e 227 fMergeExcludes(other.fMergeExcludes),
f965131e 228 fIncludePath(other.fIncludePath),
bb885a9e 229 fCloseSE(other.fCloseSE),
0df6ccf2 230 fFriendChainName(other.fFriendChainName),
c6cb3634 231 fJobTag(other.fJobTag),
648174cf 232 fOutputSingle(other.fOutputSingle),
5fce53f4 233 fRunPrefix(other.fRunPrefix),
3bdcb562 234 fProofCluster(other.fProofCluster),
235 fProofDataSet(other.fProofDataSet),
236 fFileForTestMode(other.fFileForTestMode),
237 fRootVersionForProof(other.fRootVersionForProof),
238 fAliRootMode(other.fAliRootMode),
4e5c5506 239 fInputFiles(0),
240 fPackages(0)
c57f56b7 241{
242// Copy ctor.
243 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 244 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 245 fRunRange[0] = other.fRunRange[0];
246 fRunRange[1] = other.fRunRange[1];
c57f56b7 247 if (other.fInputFiles) {
248 fInputFiles = new TObjArray();
249 TIter next(other.fInputFiles);
250 TObject *obj;
251 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
252 fInputFiles->SetOwner();
253 }
4e5c5506 254 if (other.fPackages) {
255 fPackages = new TObjArray();
256 TIter next(other.fPackages);
257 TObject *obj;
258 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
259 fPackages->SetOwner();
260 }
c57f56b7 261}
262
263//______________________________________________________________________________
264AliAnalysisAlien::~AliAnalysisAlien()
265{
266// Destructor.
267 if (fGridJDL) delete fGridJDL;
0f389141 268 if (fMergingJDL) delete fMergingJDL;
c57f56b7 269 if (fInputFiles) delete fInputFiles;
4e5c5506 270 if (fPackages) delete fPackages;
c57f56b7 271}
272
273//______________________________________________________________________________
274AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
275{
276// Assignment.
277 if (this != &other) {
278 AliAnalysisGrid::operator=(other);
279 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 280 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 281 fPrice = other.fPrice;
282 fTTL = other.fTTL;
283 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
284 fMaxInitFailed = other.fMaxInitFailed;
285 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 286 fNtestFiles = other.fNtestFiles;
a3e84053 287 fNrunsPerMaster = other.fNrunsPerMaster;
288 fMaxMergeFiles = other.fMaxMergeFiles;
289 fNsubmitted = other.fNsubmitted;
290 fProductionMode = other.fProductionMode;
cd11251e 291 fOutputToRunNo = other.fOutputToRunNo;
0f389141 292 fMergeViaJDL = other.fMergeViaJDL;
293 fFastReadOption = other.fFastReadOption;
294 fOverwriteMode = other.fOverwriteMode;
149d288c 295 fNreplicas = other.fNreplicas;
3bdcb562 296 fNproofWorkers = other.fNproofWorkers;
27734f0e 297 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
3bdcb562 298 fProofReset = other.fProofReset;
c57f56b7 299 fRunNumbers = other.fRunNumbers;
300 fExecutable = other.fExecutable;
0a1c1f7f 301 fExecutableCommand = other.fExecutableCommand;
c57f56b7 302 fArguments = other.fArguments;
631c0b05 303 fExecutableArgs = other.fExecutableArgs;
c57f56b7 304 fAnalysisMacro = other.fAnalysisMacro;
305 fAnalysisSource = other.fAnalysisSource;
d5c6455a 306 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 307 fAdditionalLibs = other.fAdditionalLibs;
308 fSplitMode = other.fSplitMode;
309 fAPIVersion = other.fAPIVersion;
310 fROOTVersion = other.fROOTVersion;
311 fAliROOTVersion = other.fAliROOTVersion;
648174cf 312 fExternalPackages = other.fExternalPackages;
c57f56b7 313 fUser = other.fUser;
314 fGridWorkingDir = other.fGridWorkingDir;
315 fGridDataDir = other.fGridDataDir;
316 fDataPattern = other.fDataPattern;
317 fGridOutputDir = other.fGridOutputDir;
318 fOutputArchive = other.fOutputArchive;
319 fOutputFiles = other.fOutputFiles;
320 fInputFormat = other.fInputFormat;
e7c71df0 321 fDatasetName = other.fDatasetName;
c57f56b7 322 fJDLName = other.fJDLName;
bb885a9e 323 fMergeExcludes = other.fMergeExcludes;
f965131e 324 fIncludePath = other.fIncludePath;
bb885a9e 325 fCloseSE = other.fCloseSE;
0df6ccf2 326 fFriendChainName = other.fFriendChainName;
c6cb3634 327 fJobTag = other.fJobTag;
648174cf 328 fOutputSingle = other.fOutputSingle;
5fce53f4 329 fRunPrefix = other.fRunPrefix;
3bdcb562 330 fProofCluster = other.fProofCluster;
331 fProofDataSet = other.fProofDataSet;
332 fFileForTestMode = other.fFileForTestMode;
333 fRootVersionForProof = other.fRootVersionForProof;
334 fAliRootMode = other.fAliRootMode;
c57f56b7 335 if (other.fInputFiles) {
336 fInputFiles = new TObjArray();
337 TIter next(other.fInputFiles);
338 TObject *obj;
339 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
340 fInputFiles->SetOwner();
341 }
4e5c5506 342 if (other.fPackages) {
343 fPackages = new TObjArray();
344 TIter next(other.fPackages);
345 TObject *obj;
346 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
347 fPackages->SetOwner();
348 }
c57f56b7 349 }
350 return *this;
351}
352
f965131e 353//______________________________________________________________________________
354void AliAnalysisAlien::AddIncludePath(const char *path)
355{
356// Add include path in the remote analysis macro.
357 TString p(path);
358 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
359 else fIncludePath += Form("-I%s ", path);
360}
361
c57f56b7 362//______________________________________________________________________________
363void AliAnalysisAlien::AddRunNumber(Int_t run)
364{
365// Add a run number to the list of runs to be processed.
366 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 367 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 368}
369
ee75cfc3 370//______________________________________________________________________________
371void AliAnalysisAlien::AddRunNumber(const char* run)
372{
373// Add a run number to the list of runs to be processed.
374 if (fRunNumbers.Length()) fRunNumbers += " ";
375 fRunNumbers += run;
376}
377
c57f56b7 378//______________________________________________________________________________
379void AliAnalysisAlien::AddDataFile(const char *lfn)
380{
381// Adds a data file to the input to be analysed. The file should be a valid LFN
382// or point to an existing file in the alien workdir.
383 if (!fInputFiles) fInputFiles = new TObjArray();
384 fInputFiles->Add(new TObjString(lfn));
385}
648174cf 386
387//______________________________________________________________________________
388void AliAnalysisAlien::AddExternalPackage(const char *package)
389{
390// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
391 if (fExternalPackages) fExternalPackages += " ";
392 fExternalPackages += package;
393}
394
c57f56b7 395//______________________________________________________________________________
396Bool_t AliAnalysisAlien::Connect()
397{
398// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
399 if (gGrid && gGrid->IsConnected()) return kTRUE;
c57f56b7 400 if (!gGrid) {
401 Info("Connect", "Trying to connect to AliEn ...");
402 TGrid::Connect("alien://");
403 }
404 if (!gGrid || !gGrid->IsConnected()) {
405 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
406 return kFALSE;
407 }
408 fUser = gGrid->GetUser();
409 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
410 return kTRUE;
411}
412
413//______________________________________________________________________________
414void AliAnalysisAlien::CdWork()
415{
416// Check validity of alien workspace. Create directory if possible.
417 if (!Connect()) {
418 Error("CdWork", "Alien connection required");
419 return;
420 }
421 TString homedir = gGrid->GetHomeDirectory();
422 TString workdir = homedir + fGridWorkingDir;
923e2ca5 423 if (DirectoryExists(workdir)) {
424 gGrid->Cd(workdir);
425 return;
426 }
427 // Work directory not existing - create it
428 gGrid->Cd(homedir);
b93f8109 429 if (gGrid->Mkdir(workdir, "-p")) {
923e2ca5 430 gGrid->Cd(fGridWorkingDir);
431 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
432 } else {
433 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
434 workdir.Data(), homedir.Data());
435 fGridWorkingDir = "";
436 }
c57f56b7 437}
438
348be253 439//______________________________________________________________________________
440Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
441{
442// Check if file copying is possible.
443 if (!Connect()) {
444 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
445 return kFALSE;
446 }
d3339be3 447 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
448 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 449 // Check if alien_CLOSE_SE is defined
450 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
451 if (!closeSE.IsNull()) {
452 Info("CheckFileCopy", "Your current close storage is pointing to: \
453 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
454 } else {
455 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
456 }
457 // Check if grid directory exists.
458 if (!DirectoryExists(alienpath)) {
459 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
460 return kFALSE;
461 }
d3339be3 462 TFile f("plugin_test_copy", "RECREATE");
348be253 463 // User may not have write permissions to current directory
464 if (f.IsZombie()) {
465 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
466 gSystem->WorkingDirectory());
467 return kFALSE;
468 }
469 f.Close();
d3339be3 470 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
471 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
472 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 473 \n# 1. Make sure you have write permissions there. If this is the case: \
474 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
475 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
476 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
477 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
478 gSystem->Unlink(f.GetName());
479 return kFALSE;
480 }
481 gSystem->Unlink(f.GetName());
d3339be3 482 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
483 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 484 return kTRUE;
485}
486
c57f56b7 487//______________________________________________________________________________
488Bool_t AliAnalysisAlien::CheckInputData()
489{
490// Check validity of input data. If necessary, create xml files.
a8739e8a 491 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 492 if (!fGridDataDir.Length()) {
493 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
494 return kFALSE;
495 }
496 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
23329835 497 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
498 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
d2a409b2 499 return kTRUE;
c57f56b7 500 }
501 // Process declared files
a2f5fc01 502 Bool_t isCollection = kFALSE;
503 Bool_t isXml = kFALSE;
504 Bool_t useTags = kFALSE;
c57f56b7 505 Bool_t checked = kFALSE;
506 CdWork();
507 TString file;
508 TString workdir = gGrid->GetHomeDirectory();
509 workdir += fGridWorkingDir;
510 if (fInputFiles) {
511 TObjString *objstr;
512 TIter next(fInputFiles);
513 while ((objstr=(TObjString*)next())) {
514 file = workdir;
515 file += "/";
516 file += objstr->GetString();
517 // Store full lfn path
518 if (FileExists(file)) objstr->SetString(file);
519 else {
520 file = objstr->GetName();
521 if (!FileExists(objstr->GetName())) {
522 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
523 objstr->GetName(), workdir.Data());
524 return kFALSE;
525 }
526 }
527 Bool_t iscoll, isxml, usetags;
528 CheckDataType(file, iscoll, isxml, usetags);
529 if (!checked) {
530 checked = kTRUE;
a2f5fc01 531 isCollection = iscoll;
532 isXml = isxml;
533 useTags = usetags;
534 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 535 } else {
a2f5fc01 536 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 537 Error("CheckInputData", "Some conflict was found in the types of inputs");
538 return kFALSE;
539 }
540 }
541 }
542 }
543 // Process requested run numbers
a8739e8a 544 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 545 // Check validity of alien data directory
546 if (!fGridDataDir.Length()) {
547 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
548 return kFALSE;
549 }
923e2ca5 550 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 551 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
552 return kFALSE;
553 }
a2f5fc01 554 if (isCollection) {
c57f56b7 555 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
556 return kFALSE;
557 }
558
a2f5fc01 559 if (checked && !isXml) {
c57f56b7 560 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
561 return kFALSE;
562 }
563 // Check validity of run number(s)
564 TObjArray *arr;
565 TObjString *os;
319593fb 566 Int_t nruns = 0;
904f9f5f 567 TString schunk, schunk2;
c57f56b7 568 TString path;
569 if (!checked) {
570 checked = kTRUE;
a2f5fc01 571 useTags = fDataPattern.Contains("tag");
572 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 573 }
a2f5fc01 574 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 575 Error("CheckInputData", "Cannot mix input files using/not using tags");
576 return kFALSE;
577 }
578 if (fRunNumbers.Length()) {
a8739e8a 579 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 580 arr = fRunNumbers.Tokenize(" ");
581 TIter next(arr);
582 while ((os=(TObjString*)next())) {
583 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 584 if (!DirectoryExists(path)) {
585 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 586 continue;
c57f56b7 587 }
588 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
589 TString msg = "\n##### file: ";
590 msg += path;
591 msg += " type: xml_collection;";
a2f5fc01 592 if (useTags) msg += " using_tags: Yes";
c57f56b7 593 else msg += " using_tags: No";
23329835 594 Info("CheckDataType", "%s", msg.Data());
319593fb 595 if (fNrunsPerMaster<2) {
d2a409b2 596 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 597 } else {
598 nruns++;
599 if (((nruns-1)%fNrunsPerMaster) == 0) {
600 schunk = os->GetString();
601 }
602 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
603 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 604 AddDataFile(schunk);
319593fb 605 }
c57f56b7 606 }
607 delete arr;
a8739e8a 608 } else {
609 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
610 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 611 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 612 if (!DirectoryExists(path)) {
613// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 614 continue;
615 }
5fce53f4 616 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 617 TString msg = "\n##### file: ";
618 msg += path;
619 msg += " type: xml_collection;";
a2f5fc01 620 if (useTags) msg += " using_tags: Yes";
a8739e8a 621 else msg += " using_tags: No";
23329835 622 Info("CheckDataType", "%s", msg.Data());
319593fb 623 if (fNrunsPerMaster<2) {
5fce53f4 624 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 625 } else {
626 nruns++;
627 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 628 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 629 }
904f9f5f 630 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 631 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 632 schunk += schunk2;
d2a409b2 633 AddDataFile(schunk);
319593fb 634 }
a8739e8a 635 }
904f9f5f 636 if (!fInputFiles) {
637 schunk += schunk2;
638 AddDataFile(schunk);
639 }
c57f56b7 640 }
641 return kTRUE;
642}
643
644//______________________________________________________________________________
645Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
646{
647// Create dataset for the grid data directory + run number.
f9e1936d 648 if (TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 649 if (!Connect()) {
650 Error("CreateDataset", "Cannot create dataset with no grid connection");
651 return kFALSE;
652 }
653
654 // Cd workspace
655 CdWork();
656 TString workdir = gGrid->GetHomeDirectory();
657 workdir += fGridWorkingDir;
658
659 // Compose the 'find' command arguments
660 TString command;
661 TString options = "-x collection ";
bb885a9e 662 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 663 TString conditions = "";
664
665 TString file;
666 TString path;
319593fb 667 Int_t nruns = 0;
904f9f5f 668 TString schunk, schunk2;
ab254fd1 669 TGridCollection *cbase=0, *cadd=0;
d2a409b2 670 if (!fRunNumbers.Length() && !fRunRange[0]) {
671 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
672 // Make a single data collection from data directory.
673 path = fGridDataDir;
923e2ca5 674 if (!DirectoryExists(path)) {
d2a409b2 675 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
676 return kFALSE;
677 }
923e2ca5 678// CdWork();
d2a409b2 679 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
680 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 681 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 682 command = "find ";
683 command += options;
684 command += path;
685 command += " ";
686 command += pattern;
687 command += conditions;
84fcd93f 688 printf("command: %s\n", command.Data());
d2a409b2 689 TGridResult *res = gGrid->Command(command);
690 if (res) delete res;
691 // Write standard output to file
692 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 693 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
694 Bool_t nullFile = kFALSE;
695 if (!hasGrep) {
d3339be3 696 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
697 } else {
a2f5fc01 698 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
699 if (nullFile) {
defd7a3a 700 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
701 return kFALSE;
702 }
d3339be3 703 }
0f389141 704 }
705 Bool_t fileExists = FileExists(file);
706 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 707 // Copy xml file to alien space
0f389141 708 if (fileExists) gGrid->Rm(file);
d2a409b2 709 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
710 if (!FileExists(file)) {
711 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
712 return kFALSE;
713 }
714 // Update list of files to be processed.
715 }
716 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
717 return kTRUE;
718 }
c57f56b7 719 // Several runs
a2f5fc01 720 Bool_t nullResult = kTRUE;
a8739e8a 721 if (fRunNumbers.Length()) {
722 TObjArray *arr = fRunNumbers.Tokenize(" ");
723 TObjString *os;
724 TIter next(arr);
725 while ((os=(TObjString*)next())) {
726 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 727 if (!DirectoryExists(path)) continue;
728// CdWork();
a8739e8a 729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", os->GetString().Data());
319593fb 731 // If local collection file does not exist, create it via 'find' command.
d3339be3 732 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 733 command = "find ";
734 command += options;
735 command += path;
736 command += pattern;
737 command += conditions;
738 TGridResult *res = gGrid->Command(command);
739 if (res) delete res;
740 // Write standard output to file
741 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 742 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
743 Bool_t nullFile = kFALSE;
744 if (!hasGrep) {
d3339be3 745 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
746 } else {
a2f5fc01 747 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
748 if (nullFile) {
defd7a3a 749 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
750 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
751 continue;
752 }
d3339be3 753 }
a2f5fc01 754 nullResult = kFALSE;
d3339be3 755 }
a8739e8a 756 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 757 // Check if there is one run per master job.
758 if (fNrunsPerMaster<2) {
759 if (FileExists(file)) {
0f389141 760 if (fOverwriteMode) gGrid->Rm(file);
761 else {
762 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
763 continue;
764 }
319593fb 765 }
766 // Copy xml file to alien space
767 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
768 if (!FileExists(file)) {
769 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
770 delete arr;
771 return kFALSE;
772 }
773 } else {
774 nruns++;
775 if (((nruns-1)%fNrunsPerMaster) == 0) {
776 schunk = os->GetString();
777 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
778 } else {
779 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 780 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 781 cbase->Add(cadd);
782 delete cadd;
783 }
784 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
785 continue;
786 }
787 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 788 if (FileExists(schunk)) {
789 if (fOverwriteMode) gGrid->Rm(file);
790 else {
791 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
792 continue;
793 }
319593fb 794 }
84fcd93f 795 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 796 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 797 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 798 if (!FileExists(schunk)) {
799 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
800 delete arr;
801 return kFALSE;
802 }
d3339be3 803 }
a8739e8a 804 }
805 delete arr;
a2f5fc01 806 if (nullResult) {
d3339be3 807 Error("CreateDataset", "No valid dataset corresponding to the query!");
808 return kFALSE;
809 }
a8739e8a 810 } else {
811 // Process a full run range.
812 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 813 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 814 if (!DirectoryExists(path)) continue;
815// CdWork();
a8739e8a 816 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 817 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
0f389141 818 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
819 if (fOverwriteMode) gGrid->Rm(file);
820 else {
821 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
822 continue;
823 }
a8739e8a 824 }
319593fb 825 // If local collection file does not exist, create it via 'find' command.
d3339be3 826 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 827 command = "find ";
828 command += options;
829 command += path;
830 command += pattern;
831 command += conditions;
832 TGridResult *res = gGrid->Command(command);
833 if (res) delete res;
834 // Write standard output to file
835 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 836 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
837 Bool_t nullFile = kFALSE;
838 if (!hasGrep) {
d3339be3 839 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
840 } else {
a2f5fc01 841 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
842 if (nullFile) {
defd7a3a 843 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
844 continue;
845 }
d3339be3 846 }
a2f5fc01 847 nullResult = kFALSE;
319593fb 848 }
a8739e8a 849 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 850 // Check if there is one run per master job.
851 if (fNrunsPerMaster<2) {
852 if (FileExists(file)) {
0f389141 853 if (fOverwriteMode) gGrid->Rm(file);
854 else {
855 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
856 continue;
857 }
319593fb 858 }
859 // Copy xml file to alien space
860 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
861 if (!FileExists(file)) {
862 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
863 return kFALSE;
864 }
865 } else {
866 nruns++;
95e5b448 867 // Check if the collection for the chunk exist locally.
868 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 869 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
870 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
871 else continue;
872 }
84fcd93f 873 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 874 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 875 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 876 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
877 } else {
878 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
879 cbase->Add(cadd);
880 delete cadd;
881 }
904f9f5f 882 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
883 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 884 continue;
885 }
904f9f5f 886 schunk = schunk2;
319593fb 887 if (FileExists(schunk)) {
0f389141 888 if (fOverwriteMode) gGrid->Rm(schunk);
889 else {
890 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
891 continue;
892 }
319593fb 893 }
84fcd93f 894 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 895 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 896 if (FileExists(schunk)) {
0f389141 897 if (fOverwriteMode) gGrid->Rm(schunk);
898 else {
899 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
900 continue;
901 }
95e5b448 902 }
319593fb 903 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
904 if (!FileExists(schunk)) {
905 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
906 return kFALSE;
907 }
908 }
c57f56b7 909 }
a2f5fc01 910 if (nullResult) {
d3339be3 911 Error("CreateDataset", "No valid dataset corresponding to the query!");
912 return kFALSE;
913 }
a8739e8a 914 }
c57f56b7 915 return kTRUE;
916}
917
918//______________________________________________________________________________
919Bool_t AliAnalysisAlien::CreateJDL()
920{
921// Generate a JDL file according to current settings. The name of the file is
922// specified by fJDLName.
923 Bool_t error = kFALSE;
924 TObjArray *arr = 0;
925 Bool_t copy = kTRUE;
926 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
927 Bool_t generate = kTRUE;
928 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
929 if (!Connect()) {
930 Error("CreateJDL", "Alien connection required");
931 return kFALSE;
932 }
933 // Check validity of alien workspace
934 CdWork();
935 TString workdir = gGrid->GetHomeDirectory();
936 workdir += fGridWorkingDir;
937 if (generate) {
938 TObjString *os;
939 if (!fInputFiles) {
940 Error("CreateJDL()", "Define some input files for your analysis.");
941 error = kTRUE;
942 }
943 // Compose list of input files
944 // Check if output files were defined
945 if (!fOutputFiles.Length()) {
946 Error("CreateJDL", "You must define at least one output file");
947 error = kTRUE;
948 }
949 // Check if an output directory was defined and valid
950 if (!fGridOutputDir.Length()) {
951 Error("CreateJDL", "You must define AliEn output directory");
952 error = kTRUE;
953 } else {
d2a409b2 954 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 955 if (!DirectoryExists(fGridOutputDir)) {
b93f8109 956 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
c57f56b7 957 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
958 } else {
959 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 960 // error = kTRUE;
c57f56b7 961 }
962 }
963 gGrid->Cd(workdir);
964 }
965 // Exit if any error up to now
966 if (error) return kFALSE;
967 // Set JDL fields
0f389141 968 if (!fUser.IsNull()) {
969 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
970 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
971 }
972 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
973 TString mergeExec = fExecutable;
974 mergeExec.ReplaceAll(".sh", "_merge.sh");
975 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
976 mergeExec.ReplaceAll(".sh", ".C");
977 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 978 if (!fArguments.IsNull())
979 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
7c2cd90a 980 fMergingJDL->SetArguments("$1 $2 $3");
1f0d1ca2 981 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
982 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
983 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
984 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
985
0f389141 986 if (fMaxInitFailed > 0) {
c57f56b7 987 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 988 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
989 }
990 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 991 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 992 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
993 }
994 if (fSplitMode.Length()) {
c57f56b7 995 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 996 fGridJDL->SetDescription("Split", "We split per SE or file");
997 }
998 if (!fAliROOTVersion.IsNull()) {
999 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1000 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1001 }
1002 if (!fROOTVersion.IsNull()) {
c57f56b7 1003 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 1004 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1005 }
1006 if (!fAPIVersion.IsNull()) {
c57f56b7 1007 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 1008 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1009 }
648174cf 1010 if (!fExternalPackages.IsNull()) {
1011 arr = fExternalPackages.Tokenize(" ");
1012 TIter next(arr);
1013 while ((os=(TObjString*)next())) {
1014 TString pkgname = os->GetString();
1015 Int_t index = pkgname.Index("::");
1016 TString pkgversion = pkgname(index+2, pkgname.Length());
1017 pkgname.Remove(index);
1018 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 1019 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 1020 }
1021 delete arr;
1022 }
0f389141 1023 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1024 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1025 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 1026 TString analysisFile = fExecutable;
1027 analysisFile.ReplaceAll(".sh", ".root");
1028 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 1029 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1030 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1031 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1032 if (fAdditionalLibs.Length()) {
1033 arr = fAdditionalLibs.Tokenize(" ");
1034 TIter next(arr);
1035 while ((os=(TObjString*)next())) {
1036 if (os->GetString().Contains(".so")) continue;
1037 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1038 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1039 }
1040 delete arr;
1041 }
4e5c5506 1042 if (fPackages) {
1043 TIter next(fPackages);
1044 TObject *obj;
0f389141 1045 while ((obj=next())) {
4e5c5506 1046 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1047 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1048 }
4e5c5506 1049 }
c57f56b7 1050 if (fOutputArchive.Length()) {
1051 arr = fOutputArchive.Tokenize(" ");
1052 TIter next(arr);
0f389141 1053 Bool_t first = kTRUE;
1054 const char *comment = "Files to be archived";
1055 const char *comment1 = comment;
1056 while ((os=(TObjString*)next())) {
1057 if (!first) comment = NULL;
1058 if (!os->GetString().Contains("@") && fCloseSE.Length())
1059 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1060 else
1061 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1062 first = kFALSE;
1063 }
c57f56b7 1064 delete arr;
0f389141 1065 TString outputArchive = fOutputArchive;
1066 if (!fMergeExcludes.IsNull()) {
1067 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1068 TIter next1(arr);
1069 while ((os=(TObjString*)next1())) {
0f389141 1070 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1071 outputArchive.ReplaceAll(os->GetString(),"");
1072 }
1073 delete arr;
1074 }
1075 arr = outputArchive.Tokenize(" ");
705adb3e 1076 TIter next2(arr);
0f389141 1077 comment = comment1;
1078 first = kTRUE;
705adb3e 1079 while ((os=(TObjString*)next2())) {
0f389141 1080 if (!first) comment = NULL;
0cdf65a8 1081 TString currentfile = os->GetString();
1082 currentfile.ReplaceAll(".root", "*.root");
1083 currentfile.ReplaceAll(".zip", "-Stage$2_$3.zip");
1084 if (!currentfile.Contains("@") && fCloseSE.Length())
1085 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
0f389141 1086 else
0cdf65a8 1087 fMergingJDL->AddToOutputArchive(currentfile, comment);
0f389141 1088 first = kFALSE;
1089 }
1090 delete arr;
c57f56b7 1091 }
149d288c 1092 arr = fOutputFiles.Tokenize(",");
c57f56b7 1093 TIter next(arr);
0f389141 1094 Bool_t first = kTRUE;
1095 const char *comment = "Files to be archived";
1096 const char *comment1 = comment;
43da816a 1097 while ((os=(TObjString*)next())) {
e1eaf596 1098 // Ignore ouputs in jdl that are also in outputarchive
1099 TString sout = os->GetString();
1100 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1101 if (fOutputArchive.Contains(sout)) continue;
0f389141 1102 if (!first) comment = NULL;
43da816a 1103 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1104 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1105 else
0f389141 1106 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
1107 first = kFALSE;
43da816a 1108 }
c57f56b7 1109 delete arr;
0f389141 1110 if (fOutputFiles.Length()) {
1111 TString outputFiles = fOutputFiles;
1112 if (!fMergeExcludes.IsNull()) {
1113 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1114 TIter next1(arr);
1115 while ((os=(TObjString*)next1())) {
0f389141 1116 outputFiles.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1117 outputFiles.ReplaceAll(os->GetString(),"");
1118 }
1119 delete arr;
1120 }
1121 arr = outputFiles.Tokenize(" ");
705adb3e 1122 TIter next2(arr);
0f389141 1123 comment = comment1;
1124 first = kTRUE;
705adb3e 1125 while ((os=(TObjString*)next2())) {
0f389141 1126 // Ignore ouputs in jdl that are also in outputarchive
1127 TString sout = os->GetString();
1128 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1129 if (fOutputArchive.Contains(sout)) continue;
1130 if (!first) comment = NULL;
1131 if (!os->GetString().Contains("@") && fCloseSE.Length())
1132 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1133 else
1134 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
020d7c04 1135 first = kFALSE;
0f389141 1136 }
1137 delete arr;
1138 }
1139 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1140 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
0d5d317c 1141 TString validationScript = fExecutable;
1142 validationScript.ReplaceAll(".sh", "_validation.sh");
0f389141 1143 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1144 validationScript = fExecutable;
1145 validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
1146 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1147 if (fMasterResubmitThreshold) {
1148 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1149 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1150 }
d2a409b2 1151 // Write a jdl with 2 input parameters: collection name and output dir name.
1152 WriteJDL(copy);
c57f56b7 1153 }
1154 // Copy jdl to grid workspace
a8739e8a 1155 if (copy) {
b5e4aaa7 1156 // Check if an output directory was defined and valid
1157 if (!fGridOutputDir.Length()) {
1158 Error("CreateJDL", "You must define AliEn output directory");
1159 return kFALSE;
1160 } else {
1161 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1162 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b93f8109 1163 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
b5e4aaa7 1164 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1165 } else {
1166 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1167 return kFALSE;
1168 }
1169 }
1170 gGrid->Cd(workdir);
1171 }
648174cf 1172 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1173 TString mergeJDLName = fExecutable;
1174 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1175 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1176 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1177 if (fProductionMode) {
648174cf 1178 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1179 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1180 }
648174cf 1181 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1182 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1183 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1184 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1185 if (fMergeViaJDL) {
1186 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1187 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1188 }
648174cf 1189 }
c57f56b7 1190 if (fAdditionalLibs.Length()) {
e7c71df0 1191 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1192 TObjString *os;
1193 TIter next(arr);
1194 while ((os=(TObjString*)next())) {
c57f56b7 1195 if (os->GetString().Contains(".so")) continue;
4e5c5506 1196 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1197 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1198 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1199 }
1200 delete arr;
1201 }
4e5c5506 1202 if (fPackages) {
1203 TIter next(fPackages);
1204 TObject *obj;
1205 while ((obj=next())) {
fdbbc7be 1206 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1207 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1208 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1209 }
1210 }
c57f56b7 1211 }
1212 return kTRUE;
1213}
1214
a8739e8a 1215//______________________________________________________________________________
d2a409b2 1216Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1217{
1218// Writes one or more JDL's corresponding to findex. If findex is negative,
1219// all run numbers are considered in one go (jdl). For non-negative indices
1220// they correspond to the indices in the array fInputFiles.
1221 if (!fInputFiles) return kFALSE;
1222 TObjString *os;
a8739e8a 1223 TString workdir = gGrid->GetHomeDirectory();
1224 workdir += fGridWorkingDir;
d2a409b2 1225
1226 if (!fRunNumbers.Length() && !fRunRange[0]) {
1227 // One jdl with no parameters in case input data is specified by name.
a8739e8a 1228 TIter next(fInputFiles);
1229 while ((os=(TObjString*)next()))
0f389141 1230 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()), "Input xml collections");
648174cf 1231 if (!fOutputSingle.IsNull())
0f389141 1232 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1233 else {
1234 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1235 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1236 }
a8739e8a 1237 } else {
d2a409b2 1238 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
0f389141 1239 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
cd11251e 1240 if (!fOutputSingle.IsNull()) {
0f389141 1241 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1242 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
cd11251e 1243 } else {
0f389141 1244 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
23329835 1245 fMergingJDL->SetOutputDirectory("$1", "Output directory");
cd11251e 1246 }
a8739e8a 1247 }
1248
1249
1250 // Generate the JDL as a string
1251 TString sjdl = fGridJDL->Generate();
0f389141 1252 TString sjdl1 = fMergingJDL->Generate();
a8739e8a 1253 Int_t index;
a8739e8a 1254 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1255 sjdl.ReplaceAll("(member", "\n (member");
1256 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1257 sjdl.ReplaceAll("{", "{\n ");
1258 sjdl.ReplaceAll("};", "\n};");
1259 sjdl.ReplaceAll("{\n \n", "{\n");
1260 sjdl.ReplaceAll("\n\n", "\n");
1261 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1262 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1263 sjdl1.ReplaceAll("(member", "\n (member");
1264 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1265 sjdl1.ReplaceAll("{", "{\n ");
1266 sjdl1.ReplaceAll("};", "\n};");
1267 sjdl1.ReplaceAll("{\n \n", "{\n");
1268 sjdl1.ReplaceAll("\n\n", "\n");
1269 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1270 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1271 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1272 index = sjdl.Index("JDLVariables");
1273 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
4739b2aa 1274 sjdl += "Workdirectorysize = {\"5000MB\"};";
0f389141 1275 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
7c2cd90a 1276 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", fJobTag.Data()));
1277 sjdl1.Prepend("# Generated merging jdl\n# $1 = full alien path to output directory to be merged\n# $2 = merging stage\n# $3 = merged chunk\n");
0f389141 1278 index = sjdl1.Index("JDLVariables");
1279 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
4739b2aa 1280 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
a8739e8a 1281 // Write jdl to file
a8739e8a 1282 ofstream out;
d2a409b2 1283 out.open(fJDLName.Data(), ios::out);
a8739e8a 1284 if (out.bad()) {
d2a409b2 1285 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1286 return kFALSE;
1287 }
1288 out << sjdl << endl;
0f389141 1289 TString mergeJDLName = fExecutable;
1290 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1291 if (fMergeViaJDL) {
1292 ofstream out1;
1293 out1.open(mergeJDLName.Data(), ios::out);
1294 if (out.bad()) {
1295 Error("CreateJDL", "Bad file name: %s", mergeJDLName.Data());
1296 return kFALSE;
1297 }
1298 out1 << sjdl1 << endl;
1299 }
a8739e8a 1300
1301 // Copy jdl to grid workspace
1302 if (!copy) {
1303 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1304 } else {
d2a409b2 1305 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1306 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1307 if (fProductionMode) {
b5fe9cba 1308 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1309 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1310 }
d2a409b2 1311 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1312 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1313 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1314 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1315 if (fMergeViaJDL) {
1316 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1317 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1318 }
a8739e8a 1319 }
1320 return kTRUE;
1321}
1322
c57f56b7 1323//______________________________________________________________________________
5513444a 1324Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1325{
1326// Returns true if file exists.
5513444a 1327 if (!gGrid) return kFALSE;
c57f56b7 1328 TGridResult *res = gGrid->Ls(lfn);
1329 if (!res) return kFALSE;
1330 TMap *map = dynamic_cast<TMap*>(res->At(0));
1331 if (!map) {
1332 delete res;
1333 return kFALSE;
1334 }
1335 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1336 if (!objs || !objs->GetString().Length()) {
1337 delete res;
1338 return kFALSE;
1339 }
1340 delete res;
1341 return kTRUE;
1342}
1343
923e2ca5 1344//______________________________________________________________________________
1345Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1346{
1347// Returns true if directory exists. Can be also a path.
1348 if (!gGrid) return kFALSE;
1349 // Check if dirname is a path
1350 TString dirstripped = dirname;
1351 dirstripped = dirstripped.Strip();
1352 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1353 TString dir = gSystem->BaseName(dirstripped);
1354 dir += "/";
1355 TString path = gSystem->DirName(dirstripped);
1356 TGridResult *res = gGrid->Ls(path, "-F");
1357 if (!res) return kFALSE;
1358 TIter next(res);
1359 TMap *map;
1360 TObject *obj;
1361 while ((map=dynamic_cast<TMap*>(next()))) {
1362 obj = map->GetValue("name");
1363 if (!obj) break;
1364 if (dir == obj->GetName()) {
1365 delete res;
1366 return kTRUE;
1367 }
1368 }
1369 delete res;
1370 return kFALSE;
1371}
1372
c57f56b7 1373//______________________________________________________________________________
a2f5fc01 1374void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1375{
1376// Check input data type.
a2f5fc01 1377 isCollection = kFALSE;
1378 isXml = kFALSE;
1379 useTags = kFALSE;
c57f56b7 1380 if (!gGrid) {
1381 Error("CheckDataType", "No connection to grid");
1382 return;
1383 }
a2f5fc01 1384 isCollection = IsCollection(lfn);
c57f56b7 1385 TString msg = "\n##### file: ";
1386 msg += lfn;
a2f5fc01 1387 if (isCollection) {
c57f56b7 1388 msg += " type: raw_collection;";
1389 // special treatment for collections
a2f5fc01 1390 isXml = kFALSE;
c57f56b7 1391 // check for tag files in the collection
1392 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1393 if (!res) {
1394 msg += " using_tags: No (unknown)";
23329835 1395 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1396 return;
1397 }
1398 const char* typeStr = res->GetKey(0, "origLFN");
1399 if (!typeStr || !strlen(typeStr)) {
1400 msg += " using_tags: No (unknown)";
23329835 1401 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1402 return;
1403 }
1404 TString file = typeStr;
a2f5fc01 1405 useTags = file.Contains(".tag");
1406 if (useTags) msg += " using_tags: Yes";
c57f56b7 1407 else msg += " using_tags: No";
23329835 1408 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1409 return;
1410 }
1411 TString slfn(lfn);
1412 slfn.ToLower();
a2f5fc01 1413 isXml = slfn.Contains(".xml");
1414 if (isXml) {
c57f56b7 1415 // Open xml collection and check if there are tag files inside
1416 msg += " type: xml_collection;";
1417 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1418 if (!coll) {
1419 msg += " using_tags: No (unknown)";
23329835 1420 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1421 return;
1422 }
1423 TMap *map = coll->Next();
1424 if (!map) {
1425 msg += " using_tags: No (unknown)";
23329835 1426 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1427 return;
1428 }
1429 map = (TMap*)map->GetValue("");
1430 TString file;
1431 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1432 useTags = file.Contains(".tag");
c57f56b7 1433 delete coll;
a2f5fc01 1434 if (useTags) msg += " using_tags: Yes";
c57f56b7 1435 else msg += " using_tags: No";
23329835 1436 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1437 return;
1438 }
a2f5fc01 1439 useTags = slfn.Contains(".tag");
c57f56b7 1440 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1441 else msg += " type: unknown file;";
a2f5fc01 1442 if (useTags) msg += " using_tags: Yes";
c57f56b7 1443 else msg += " using_tags: No";
23329835 1444 Info("CheckDataType", "%s", msg.Data());
c57f56b7 1445}
1446
4e5c5506 1447//______________________________________________________________________________
1448void AliAnalysisAlien::EnablePackage(const char *package)
1449{
1450// Enables a par file supposed to exist in the current directory.
1451 TString pkg(package);
1452 pkg.ReplaceAll(".par", "");
1453 pkg += ".par";
1454 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1455 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1456 return;
1457 }
1458 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1459 Info("EnablePackage", "AliEn plugin will use .par packages");
1460 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1461 if (!fPackages) {
1462 fPackages = new TObjArray();
1463 fPackages->SetOwner();
1464 }
1465 fPackages->Add(new TObjString(pkg));
1466}
1467
3bdcb562 1468//______________________________________________________________________________
1469TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1470{
1471// Make a tree from files having the location specified in fFileForTestMode.
1472// Inspired from JF's CreateESDChain.
1473 if (fFileForTestMode.IsNull()) {
1474 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1475 return NULL;
1476 }
1477 if (gSystem->AccessPathName(fFileForTestMode)) {
1478 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1479 return NULL;
1480 }
1481 // Open the file
1482 ifstream in;
1483 in.open(fFileForTestMode);
1484 Int_t count = 0;
1485 // Read the input list of files and add them to the chain
1486 TString line;
1487 TChain *chain = new TChain(treeName);
1488 while (in.good())
1489 {
1490 in >> line;
1491 if (line.IsNull()) continue;
1492 if (count++ == fNtestFiles) break;
1493 TString esdFile(line);
1494 TFile *file = TFile::Open(esdFile);
1495 if (file) {
1496 if (!file->IsZombie()) chain->Add(esdFile);
1497 file->Close();
1498 } else {
1499 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1500 }
1501 }
1502 in.close();
1503 if (!chain->GetListOfFiles()->GetEntries()) {
1504 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1505 delete chain;
1506 return NULL;
1507 }
1508// chain->ls();
1509 return chain;
1510}
1511
d2a409b2 1512//______________________________________________________________________________
1513const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1514{
1515// Get job status for all jobs with jobid>jobidstart.
1516 static char mstatus[20];
1517 mstatus[0] = '\0';
1518 nrunning = 0;
1519 nwaiting = 0;
1520 nerror = 0;
1521 ndone = 0;
1522 TGridJobStatusList *list = gGrid->Ps("");
1523 if (!list) return mstatus;
1524 Int_t nentries = list->GetSize();
1525 TGridJobStatus *status;
1526 Int_t pid;
1527 for (Int_t ijob=0; ijob<nentries; ijob++) {
1528 status = (TGridJobStatus *)list->At(ijob);
1529 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1530 if (pid<jobidstart) continue;
1531 if (pid == lastid) {
1532 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1533 }
1534 switch (status->GetStatus()) {
1535 case TGridJobStatus::kWAITING:
1536 nwaiting++; break;
1537 case TGridJobStatus::kRUNNING:
1538 nrunning++; break;
1539 case TGridJobStatus::kABORTED:
1540 case TGridJobStatus::kFAIL:
1541 case TGridJobStatus::kUNKNOWN:
1542 nerror++; break;
1543 case TGridJobStatus::kDONE:
1544 ndone++;
1545 }
1546 }
1547 list->Delete();
1548 delete list;
1549 return mstatus;
1550}
1551
c57f56b7 1552//______________________________________________________________________________
1553Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1554{
1555// Returns true if file is a collection. Functionality duplicated from
1556// TAlien::Type() because we don't want to directly depend on TAlien.
1557 if (!gGrid) {
1558 Error("IsCollection", "No connection to grid");
1559 return kFALSE;
1560 }
1561 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1562 if (!res) return kFALSE;
1563 const char* typeStr = res->GetKey(0, "type");
1564 if (!typeStr || !strlen(typeStr)) return kFALSE;
1565 if (!strcmp(typeStr, "collection")) return kTRUE;
1566 delete res;
1567 return kFALSE;
1568}
1569
fe2d7fc2 1570//______________________________________________________________________________
1571Bool_t AliAnalysisAlien::IsSingleOutput() const
1572{
1573// Check if single-ouput option is on.
1574 return (!fOutputSingle.IsNull());
1575}
1576
16a4353c 1577//______________________________________________________________________________
1578void AliAnalysisAlien::Print(Option_t *) const
1579{
1580// Print current plugin settings.
84fcd93f 1581 printf("### AliEn analysis plugin current settings ###\n");
e1c22e21 1582 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1583 if (fOverwriteMode) {
1584 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1585 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1586 }
348be253 1587 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1588 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1589 printf("= Production mode:______________________________ %d\n", fProductionMode);
1590 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1591 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1592 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1593 if (fUser.Length())
84fcd93f 1594 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1595 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1596 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1597 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1598 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1599 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1600 if (fRunNumbers.Length())
84fcd93f 1601 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1602 if (fRunRange[0])
5fce53f4 1603 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1604 if (!fRunRange[0] && !fRunNumbers.Length()) {
1605 TIter next(fInputFiles);
1606 TObject *obj;
1607 TString list;
1608 while ((obj=next())) list += obj->GetName();
84fcd93f 1609 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1610 }
1611 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1612 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1613 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1614 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1615 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1616 printf("=====================================================================\n");
1617 printf("= Job price: ___________________________________ %d\n", fPrice);
1618 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1619 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1620 if (fMaxInitFailed>0)
84fcd93f 1621 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1622 if (fMasterResubmitThreshold>0)
84fcd93f 1623 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1624 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1625 if (fNrunsPerMaster>0)
84fcd93f 1626 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1627 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1628 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1629 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1630 if (fArguments.Length())
84fcd93f 1631 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1632 if (fExecutableArgs.Length())
1633 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1634 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1635 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1636 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1637 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1638 if (fDatasetName)
84fcd93f 1639 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1640 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1641 if (fIncludePath.Data())
84fcd93f 1642 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1643 if (fCloseSE.Length())
84fcd93f 1644 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1645 if (fFriendChainName.Length())
84fcd93f 1646 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1647 if (fPackages) {
1648 TIter next(fPackages);
1649 TObject *obj;
1650 TString list;
1651 while ((obj=next())) list += obj->GetName();
84fcd93f 1652 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1653 }
1654}
1655
c57f56b7 1656//______________________________________________________________________________
1657void AliAnalysisAlien::SetDefaults()
1658{
1659// Set default values for everything. What cannot be filled will be left empty.
1660 if (fGridJDL) delete fGridJDL;
1661 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1662 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1663 fPrice = 1;
1664 fTTL = 30000;
1665 fSplitMaxInputFileNumber = 100;
1666 fMaxInitFailed = 0;
1667 fMasterResubmitThreshold = 0;
bb885a9e 1668 fNtestFiles = 10;
149d288c 1669 fNreplicas = 2;
a8739e8a 1670 fRunRange[0] = 0;
1671 fRunRange[1] = 0;
319593fb 1672 fNrunsPerMaster = 1;
16a4353c 1673 fMaxMergeFiles = 100;
c57f56b7 1674 fRunNumbers = "";
1675 fExecutable = "analysis.sh";
0a1c1f7f 1676 fExecutableCommand = "root -b -q";
c57f56b7 1677 fArguments = "";
631c0b05 1678 fExecutableArgs = "";
c57f56b7 1679 fAnalysisMacro = "myAnalysis.C";
1680 fAnalysisSource = "";
1681 fAdditionalLibs = "";
1682 fSplitMode = "se";
1683 fAPIVersion = "";
1684 fROOTVersion = "";
1685 fAliROOTVersion = "";
1686 fUser = ""; // Your alien user name
1687 fGridWorkingDir = "";
1688 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1689 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1690 fFriendChainName = "";
c57f56b7 1691 fGridOutputDir = "output";
149d288c 1692 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1693 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1694 fInputFormat = "xml-single";
1695 fJDLName = "analysis.jdl";
c6cb3634 1696 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1697 fMergeExcludes = "";
0f389141 1698 fMergeViaJDL = 0;
348be253 1699 SetUseCopy(kTRUE);
1700 SetCheckCopy(kTRUE);
149d288c 1701 SetDefaultOutputs(kTRUE);
e1c22e21 1702 fOverwriteMode = 1;
c57f56b7 1703}
1704
0f389141 1705//______________________________________________________________________________
7c2cd90a 1706Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, Bool_t submit, const char *jdl)
0f389141 1707{
7c2cd90a 1708// Static method that checks the status of merging. This can submit merging jobs that did not produced the expected
1709// output. If <submit> is false (checking) returns true only when the final merged file was found. If submit is true returns
1710// true if the jobs were successfully submitted.
1711 Int_t countOrig = 0;
1712 Int_t countStage = 0;
1713 Int_t stage = 0;
1714 Int_t i;
1715 Bool_t doneFinal = kFALSE;
1716 TBits chunksDone;
1717 TString saliendir(aliendir);
1718 TString sfilename, stmp;
1719 saliendir.ReplaceAll("//","/");
1720 saliendir = saliendir.Strip(TString::kTrailing, '/');
1721 if (!gGrid) {
1722 ::Error("GetNregisteredFiles", "You need to be connected to AliEn.");
1723 return kFALSE;
1724 }
0cdf65a8 1725 sfilename = filename;
1726 sfilename.ReplaceAll(".root", "*.root");
1727 printf("Checking directory <%s> for merged files <%s> ...\n", aliendir, sfilename.Data());
1728 TString command = Form("find %s/ *%s", saliendir.Data(), sfilename.Data());
7c2cd90a 1729 TGridResult *res = gGrid->Command(command);
1730 if (!res) {
1731 ::Error("GetNregisteredFiles","Error: No result for the find command\n");
1732 return kFALSE;
1733 }
1734 TIter nextmap(res);
1735 TMap *map = 0;
1736 while ((map=(TMap*)nextmap())) {
1737 TString turl = map->GetValue("turl")->GetName();
1738 if (!turl.Length()) {
1739 // Nothing found
1740 delete res;
1741 return kFALSE;
1742 }
1743 turl.ReplaceAll("alien://", "");
1744 turl.ReplaceAll(saliendir, "");
1745 sfilename = gSystem->BaseName(turl);
0cdf65a8 1746 turl = turl.Strip(TString::kLeading, '/');
7c2cd90a 1747 // Now check to what the file corresponds to:
1748 // original output - aliendir/%03d/filename
1749 // merged file (which stage) - aliendir/filename-Stage%02d_%04d
1750 // final merged file - aliendir/filename
1751 if (sfilename == turl) {
1752 if (sfilename == filename) {
1753 doneFinal = kTRUE;
1754 } else {
1755 // check stage
1756 Int_t index = sfilename.Index("Stage");
1757 if (index<0) continue;
1758 stmp = sfilename(index+5,2);
1759 Int_t istage = atoi(stmp);
1760 stmp = sfilename(index+8,4);
1761 Int_t ijob = atoi(stmp);
1762 if (istage<stage) continue; // Ignore lower stages
1763 if (istage>stage) {
1764 countStage = 0;
1765 chunksDone.ResetAllBits();
1766 stage = istage;
1767 }
1768 countStage++;
1769 chunksDone.SetBitNumber(ijob);
1770 }
1771 } else {
1772 countOrig++;
1773 }
1774 if (doneFinal) {
1775 delete res;
0cdf65a8 1776 printf("=> Removing files from previous stages...\n");
1777 gGrid->Rm(Form("%s/*Stage*.root", aliendir));
7a62d541 1778 for (i=1; i<stage; i++)
1779 gGrid->Rm(Form("%s/*Stage%d*.zip", aliendir, i));
7c2cd90a 1780 return kTRUE;
1781 }
1782 }
1783 delete res;
1784 // Compute number of jobs that were submitted for the current stage
1785 Int_t ntotstage = countOrig;
7ae54d70 1786 for (i=1; i<=stage; i++) {
1787 if (ntotstage%nperchunk) ntotstage = (ntotstage/nperchunk)+1;
1788 else ntotstage = (ntotstage/nperchunk);
1789 }
7c2cd90a 1790 // Now compare with the number of set bits in the chunksDone array
1791 Int_t nmissing = (stage>0)?(ntotstage - countStage):0;
1792 // Print the info
1793 printf("*** Found %d original files\n", countOrig);
1794 if (stage==0) printf("*** No merging completed so far.\n");
1795 else printf("*** Found %d out of %d files merged for stage %d\n", countStage, ntotstage, stage);
1796 if (nmissing) printf("*** Number of merged files missing for this stage: %d -> check merging job completion\n", nmissing);
7c2cd90a 1797 if (!submit) return doneFinal;
1798 // Sumbit merging jobs for all missing chunks for the current stage.
1799 TString query = Form("submit %s %s", jdl, aliendir);
1800 Int_t ichunk = -1;
1801 if (nmissing) {
1802 for (i=0; i<nmissing; i++) {
1803 ichunk = chunksDone.FirstNullBit(ichunk+1);
1804 Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage, ichunk));
1805 if (!jobId) return kFALSE;
1806 }
1807 return kTRUE;
1808 }
1809 // Submit next stage of merging
1810 if (stage==0) countStage = countOrig;
7ae54d70 1811 Int_t nchunks = (countStage/nperchunk);
1812 if (countStage%nperchunk) nchunks += 1;
7c2cd90a 1813 for (i=0; i<nchunks; i++) {
1814 Int_t jobId = SubmitSingleJob(Form("%s %d %d", query.Data(), stage+1, i));
1815 if (!jobId) return kFALSE;
1816 }
1817 return kTRUE;
1818}
1819
1820//______________________________________________________________________________
1821Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1822{
1823// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1824 if (!gGrid) return 0;
1825 printf("=> %s ------> ",query);
1826 TGridResult *res = gGrid->Command(query);
1827 if (!res) return 0;
1828 TString jobId = res->GetKey(0,"jobId");
1829 delete res;
1830 if (jobId.IsNull()) {
1831 printf("submission failed. Reason:\n");
1832 gGrid->Stdout();
1833 gGrid->Stderr();
1834 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1835 return 0;
1836 }
1837 printf(" Job id: %s\n", jobId.Data());
1838 return atoi(jobId);
1839}
1840
1841//______________________________________________________________________________
1842Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage, Int_t ichunk)
1843{
1844// Merge given output files from basedir. The file merger will merge nmaxmerge
1845// files in a group. Merging can be done in stages:
1846// stage=0 : will merge all existing files in a single stage
1847// stage=1 : does a find command for all files that do NOT contain the string "Stage".
1848// If their number is bigger that nmaxmerge, only the files from
1849// ichunk*nmaxmerge to ichunk*(nmaxmerge+1)-1 will get merged as output_stage_<ichunk>
1850// stage=n : does a find command for files named <output>Stage<stage-1>_*. If their number is bigger than
1851// nmaxmerge, merge just the chunk ichunk, otherwise write the merged output to the file
1852// named <output>.
a2f5fc01 1853 TString outputFile = output;
0f389141 1854 TString command;
a2f5fc01 1855 TString outputChunk;
1856 TString previousChunk = "";
1857 Int_t countChunk = 0;
1858 Int_t countZero = nmaxmerge;
0f389141 1859 Bool_t merged = kTRUE;
a2f5fc01 1860 Int_t index = outputFile.Index("@");
1861 if (index > 0) outputFile.Remove(index);
7c2cd90a 1862 TString inputFile = outputFile;
1863 if (stage>1) inputFile.ReplaceAll(".root", Form("-Stage%02d_*.root", stage-1));
1864 command = Form("find %s/ *%s", basedir, inputFile.Data());
0f389141 1865 printf("command: %s\n", command.Data());
1866 TGridResult *res = gGrid->Command(command);
1867 if (!res) {
7c2cd90a 1868 ::Error("MergeOutput","No result for the find command\n");
0f389141 1869 return kFALSE;
1870 }
1871
1872 TFileMerger *fm = 0;
1873 TIter nextmap(res);
1874 TMap *map = 0;
7c2cd90a 1875 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
a2f5fc01 1876 outputChunk = outputFile;
1877 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 1878 // Check for existent temporary merge files
e1c22e21 1879 // Check overwrite mode and remove previous partial results if needed
7c2cd90a 1880 // Preserve old merging functionality for stage 0.
1881 if (stage==0) {
1882 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
1883 while (1) {
1884 // Skip as many input files as in a chunk
1885 for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
1886 if (!map) {
1887 ::Error("MergeOutput", "Cannot resume merging for <%s>, nentries=%d", outputFile.Data(), res->GetSize());
1888 delete res;
1889 return kFALSE;
1890 }
1891 outputChunk = outputFile;
1892 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
1893 countChunk++;
1894 if (gSystem->AccessPathName(outputChunk)) continue;
1895 // Merged file with chunks up to <countChunk> found
1896 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
1897 previousChunk = outputChunk;
1898 break;
1899 }
1900 }
1901 countZero = nmaxmerge;
1902
1903 while ((map=(TMap*)nextmap())) {
1904 // Loop 'find' results and get next LFN
1905 if (countZero == nmaxmerge) {
1906 // First file in chunk - create file merger and add previous chunk if any.
1907 fm = new TFileMerger(kFALSE);
1908 fm->SetFastMethod(kTRUE);
1909 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
1910 outputChunk = outputFile;
1911 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
1912 }
1913 // If last file found, put merged results in the output file
1914 if (map == res->Last()) outputChunk = outputFile;
1915 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1916 if (!objs || !objs->GetString().Length()) {
1917 // Nothing found - skip this output
0f389141 1918 delete res;
7c2cd90a 1919 delete fm;
0f389141 1920 return kFALSE;
7c2cd90a 1921 }
1922 // Add file to be merged and decrement chunk counter.
1923 fm->AddFile(objs->GetString());
1924 countZero--;
1925 if (countZero==0 || map == res->Last()) {
1926 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1927 // Nothing found - skip this output
1928 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
1929 delete res;
1930 delete fm;
1931 return kFALSE;
1932 }
1933 fm->OutputFile(outputChunk);
1934 // Merge the outputs, then go to next chunk
1935 if (!fm->Merge()) {
1936 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
1937 delete res;
1938 delete fm;
1939 return kFALSE;
1940 } else {
1941 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
1942 gSystem->Unlink(previousChunk);
1943 }
1944 if (map == res->Last()) {
1945 delete res;
1946 delete fm;
1947 break;
1948 }
1949 countChunk++;
1950 countZero = nmaxmerge;
1951 previousChunk = outputChunk;
0f389141 1952 }
0f389141 1953 }
7c2cd90a 1954 return merged;
1955 }
1956 // Merging stage different than 0.
1957 // Move to the begining of the requested chunk.
1958 outputChunk = outputFile;
1959 if (nmaxmerge < res->GetSize()) {
1960 if (ichunk*nmaxmerge >= res->GetSize()) {
1961 ::Error("MergeOutput", "Cannot merge merge chunk %d grouping %d files from %d total.", ichunk, nmaxmerge, res->GetSize());
1962 delete res;
1963 return kFALSE;
1964 }
1965 for (Int_t counter=0; counter<ichunk*nmaxmerge; counter++) map = (TMap*)nextmap();
1966 outputChunk.ReplaceAll(".root", Form("-Stage%02d_%04d.root", stage, ichunk));
1967 }
1968 countZero = nmaxmerge;
1969 fm = new TFileMerger(kFALSE);
1970 fm->SetFastMethod(kTRUE);
0f389141 1971 while ((map=(TMap*)nextmap())) {
7c2cd90a 1972 // Loop 'find' results and get next LFN
0f389141 1973 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1974 if (!objs || !objs->GetString().Length()) {
1975 // Nothing found - skip this output
1976 delete res;
1977 delete fm;
1978 return kFALSE;
1979 }
1980 // Add file to be merged and decrement chunk counter.
1981 fm->AddFile(objs->GetString());
a2f5fc01 1982 countZero--;
7c2cd90a 1983 if (countZero==0) break;
1984 }
1985 delete res;
1986 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1987 // Nothing found - skip this output
1988 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
1989 delete fm;
1990 return kFALSE;
1991 }
1992 fm->OutputFile(outputChunk);
1993 // Merge the outputs
1994 if (!fm->Merge()) {
1995 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
1996 delete fm;
1997 return kFALSE;
1998 } else {
1999 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
0f389141 2000 }
7c2cd90a 2001 delete fm;
2002 return kTRUE;
0f389141 2003}
2004
c57f56b7 2005//______________________________________________________________________________
2006Bool_t AliAnalysisAlien::MergeOutputs()
2007{
2008// Merge analysis outputs existing in the AliEn space.
2009 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2010 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2011 if (!Connect()) {
2012 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2013 return kFALSE;
0f389141 2014 }
c9e8f7fd 2015 if (fMergeViaJDL) {
2016 if (!TestBit(AliAnalysisGrid::kMerge)) {
2017 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2018 return kFALSE;
2019 }
2020 if (fProductionMode) {
2021 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2022 return kFALSE;
2023 }
0f389141 2024 Info("MergeOutputs", "Submitting merging JDL");
a03be957 2025 if (!SubmitMerging()) return kFALSE;
0f389141 2026 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 2027 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 2028 return kFALSE;
c57f56b7 2029 }
2030 // Get the output path
d2a409b2 2031 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 2032 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 2033 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2034 return kFALSE;
2035 }
2036 if (!fOutputFiles.Length()) {
2037 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2038 return kFALSE;
0f389141 2039 }
2040 // Check if fast read option was requested
e1c22e21 2041 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2042 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 2043 if (fFastReadOption) {
d3339be3 2044 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2045 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
2046 gEnv->SetValue("XNet.ConnectTimeout",10);
2047 gEnv->SetValue("XNet.RequestTimeout",10);
0f389141 2048 gEnv->SetValue("XNet.MaxRedirectCount",2);
d3339be3 2049 gEnv->SetValue("XNet.ReconnectTimeout",10);
0f389141 2050 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 2051 }
e8b839ab 2052 // Make sure we change the temporary directory
2053 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 2054 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2055 TIter next(list);
2056 TObjString *str;
a2f5fc01 2057 TString outputFile;
c57f56b7 2058 Bool_t merged = kTRUE;
2059 while((str=(TObjString*)next())) {
a2f5fc01 2060 outputFile = str->GetString();
2061 Int_t index = outputFile.Index("@");
2062 if (index > 0) outputFile.Remove(index);
2063 TString outputChunk = outputFile;
2064 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 2065 // Skip already merged outputs
a2f5fc01 2066 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 2067 if (fOverwriteMode) {
a2f5fc01 2068 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2069 gSystem->Unlink(outputFile);
2070 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2071 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2072 outputChunk.Data());
2073 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2074 }
2075 } else {
a2f5fc01 2076 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 2077 continue;
2078 }
2079 } else {
a2f5fc01 2080 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 2081 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 2082 outputChunk.Data());
2083 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 2084 }
2085 }
bb885a9e 2086 if (fMergeExcludes.Length() &&
a2f5fc01 2087 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 2088 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 2089 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 2090 if (!merged) {
2091 Error("MergeOutputs", "Terminate() will NOT be executed");
2092 return kFALSE;
ff07ec61 2093 }
2094 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2095 if (fileOpened) fileOpened->Close();
c57f56b7 2096 }
0f389141 2097 return kTRUE;
c57f56b7 2098}
2099
bb885a9e 2100//______________________________________________________________________________
2101void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2102{
2103// Use the output files connected to output containers from the analysis manager
2104// rather than the files defined by SetOutputFiles
2105 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
205b201f 2106 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
bb885a9e 2107 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2108}
2109
149d288c 2110//______________________________________________________________________________
2111void AliAnalysisAlien::SetOutputFiles(const char *list)
2112{
2113// Manually set the output files list.
2114// Removes duplicates. Not allowed if default outputs are not disabled.
2115 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2116 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2117 return;
2118 }
2119 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2120 fOutputFiles = "";
2121 TString slist = list;
2122 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2123 TObjArray *arr = slist.Tokenize(" ");
2124 TObjString *os;
2125 TIter next(arr);
2126 TString sout;
2127 while ((os=(TObjString*)next())) {
2128 sout = os->GetString();
2129 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2130 if (fOutputFiles.Contains(sout)) continue;
2131 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2132 fOutputFiles += sout;
2133 }
2134 delete arr;
2135}
2136
2137//______________________________________________________________________________
2138void AliAnalysisAlien::SetOutputArchive(const char *list)
2139{
2140// Manually set the output archive list. Free text - you are on your own...
2141// Not allowed if default outputs are not disabled.
2142 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2143 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2144 return;
2145 }
2146 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2147 fOutputArchive = list;
2148}
2149
2150//______________________________________________________________________________
2151void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2152{
2153// Setting a prefered output SE is not allowed anymore.
2154 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2155}
2156
5513444a 2157//______________________________________________________________________________
2158Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 2159{
2160// Start remote grid analysis.
3bdcb562 2161 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2162 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2163 if (!mgr || !mgr->IsInitialized()) {
2164 Error("StartAnalysis", "You need an initialized analysis manager for this");
2165 return kFALSE;
2166 }
2167 // Are we in PROOF mode ?
2168 if (mgr->IsProofMode()) {
2169 Info("StartAnalysis", "##### Starting PROOF analysis on via the plugin #####");
2170 if (fProofCluster.IsNull()) {
2171 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2172 return kFALSE;
2173 }
2174 if (fProofDataSet.IsNull() && !testMode) {
2175 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2176 return kFALSE;
2177 }
2178 // Set the needed environment
2179 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2180 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2181 if (fProofReset && !testMode) {
2182 if (fProofReset==1) {
2183 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2184 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2185 } else {
2186 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2187 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2188 }
2189 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2190 return kFALSE;
2191 }
2192 // Do we need to change the ROOT version ? The success of this cannot be checked.
2193 if (!fRootVersionForProof.IsNull() && !testMode) {
2194 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2195 fProofCluster.Data(), fRootVersionForProof.Data()));
2196 }
2197 // Connect to PROOF and check the status
2198 Long_t proof = 0;
27734f0e 2199 TString sworkers;
2200 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2201 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3bdcb562 2202 if (!testMode) {
27734f0e 2203 if (!sworkers.IsNull())
2204 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3bdcb562 2205 else
2206 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2207 } else {
2208 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2209 if (!proof) {
2210 Error("StartAnalysis", "Could not start PROOF in test mode");
2211 return kFALSE;
2212 }
2213 }
2214 if (!proof) {
2215 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2216 return kFALSE;
2217 }
27734f0e 2218 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2219 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3bdcb562 2220 // Is dataset existing ?
2221 if (!testMode) {
2222 TString dataset = fProofDataSet;
2223 Int_t index = dataset.Index("#");
2224 if (index>=0) dataset.Remove(index);
830acc4c 2225// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2226// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2227// return kFALSE;
2228// }
2229// Info("StartAnalysis", "Dataset %s found", dataset.Data());
3bdcb562 2230 }
2231 // Is ClearPackages() needed ?
2232 if (TestSpecialBit(kClearPackages)) {
2233 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2234 gROOT->ProcessLine("gProof->ClearPackages();");
2235 }
2236 // Is a given aliroot mode requested ?
2237 TList optionsList;
830acc4c 2238 if (!fAliRootMode.IsNull()) {
3bdcb562 2239 TString alirootMode = fAliRootMode;
2240 if (alirootMode == "default") alirootMode = "";
2241 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2242 optionsList.SetOwner();
2243 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2244 // Check the additional libs to be loaded
2245 TString extraLibs;
2246 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2247 // Parse the extra libs for .so
2248 if (fAdditionalLibs.Length()) {
2249 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2250 TIter next(list);
2251 TObjString *str;
2252 while((str=(TObjString*)next()) && str->GetString().Contains(".so")) {
830acc4c 2253 TString stmp = str->GetName();
2254 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2255 stmp.ReplaceAll(".so","");
3bdcb562 2256 if (!extraLibs.IsNull()) extraLibs += ":";
830acc4c 2257 extraLibs += stmp;
3bdcb562 2258 }
2259 if (list) delete list;
2260 }
2261 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
830acc4c 2262 // Check extra includes
2263 if (!fIncludePath.IsNull()) {
2264 TString includePath = fIncludePath;
2265 includePath.ReplaceAll(" ",":");
2266 includePath.Strip(TString::kTrailing, ':');
2267 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2268 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2269 }
27734f0e 2270 // Check if connection to grid is requested
2271 if (TestSpecialBit(kProofConnectGrid))
2272 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
830acc4c 2273 // Enable AliRoot par
2274 if (testMode) {
2275 // Enable proof lite package
2276 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2277 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2278 TNamed *obj = (TNamed*)optionsList.At(i);
2279 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2280 }
2281 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
2282 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)0x%lx);",alirootLite.Data(),(ULong_t)&optionsList))) {
2283 Info("StartAnalysis", "AliRootProofLite enabled");
2284 } else {
2285 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2286 return kFALSE;
2287 }
2288 } else {
2289 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)0x%lx);",
2290 fAliROOTVersion.Data(), (ULong_t)&optionsList))) {
2291 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2292 return kFALSE;
2293 }
27734f0e 2294 }
3bdcb562 2295 } else {
2296 if (fAdditionalLibs.Contains(".so") && !testMode) {
2297 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2298 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2299 return kFALSE;
2300 }
2301 }
2302 // Enable par files if requested
2303 if (fPackages && fPackages->GetEntries()) {
2304 TIter next(fPackages);
2305 TObject *package;
2306 while ((package=next())) {
2307 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
27734f0e 2308 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3bdcb562 2309 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2310 return kFALSE;
2311 }
2312 } else {
2313 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2314 return kFALSE;
2315 }
2316 }
2317 }
2318 // Do we need to load analysis source files ?
2319 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2320 if (fAnalysisSource.Length()) {
2321 TObjArray *list = fAnalysisSource.Tokenize(" ");
2322 TIter next(list);
2323 TObjString *str;
2324 while((str=(TObjString*)next())) {
2325 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2326 }
2327 if (list) delete list;
2328 }
830acc4c 2329 if (testMode) {
2330 // Register dataset to proof lite.
2331 if (fFileForTestMode.IsNull()) {
2332 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2333 return kFALSE;
2334 }
2335 if (gSystem->AccessPathName(fFileForTestMode)) {
2336 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2337 return kFALSE;
2338 }
2339 TFileCollection *coll = new TFileCollection();
2340 coll->AddFromFile(fFileForTestMode);
27734f0e 2341 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)0x%lx, \"OV\");", (ULong_t)coll));
2342 gROOT->ProcessLine("gProof->ShowDataSets()");
830acc4c 2343 }
3bdcb562 2344 return kTRUE;
2345 }
c57f56b7 2346
43da816a 2347 // Check if output files have to be taken from the analysis manager
2348 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
43da816a 2349 fOutputFiles = "";
2350 TIter next(mgr->GetOutputs());
2351 AliAnalysisDataContainer *output;
2352 while ((output=(AliAnalysisDataContainer*)next())) {
2353 const char *filename = output->GetFileName();
2354 if (!(strcmp(filename, "default"))) {
2355 if (!mgr->GetOutputEventHandler()) continue;
2356 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
2357 }
84fcd93f 2358 if (fOutputFiles.Contains(filename)) continue;
149d288c 2359 if (fOutputFiles.Length()) fOutputFiles += ",";
43da816a 2360 fOutputFiles += filename;
2361 }
c07b9ce2 2362 // Add extra files registered to the analysis manager
2363 if (mgr->GetExtraFiles().Length()) {
149d288c 2364 if (fOutputFiles.Length()) fOutputFiles += ",";
2365 TString extra = mgr->GetExtraFiles();
2366 extra.ReplaceAll(" ", ",");
2367 // Protection in case extra files do not exist (will it work?)
2368 extra.ReplaceAll(".root", "*.root");
2369 fOutputFiles += extra;
84fcd93f 2370 }
149d288c 2371 // Compose the output archive.
2372 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2373 fOutputArchive += Form("root_archive.zip:%s@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 2374 }
f7b1cbc2 2375// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 2376 if (TestBit(AliAnalysisGrid::kOffline)) {
2377 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2378 \n there nor any job run. You can revise the JDL and analysis \
2379 \n macro then run the same in \"submit\" mode.");
2380 } else if (TestBit(AliAnalysisGrid::kTest)) {
2381 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2382 \n dataset.");
2383 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2384 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2385 \n space and job submitted.");
2386 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2387 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 2388 if (fMergeViaJDL) CheckInputData();
5513444a 2389 return kTRUE;
c57f56b7 2390 } else {
2391 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2392 }
2393
348be253 2394 Print();
c57f56b7 2395 if (!Connect()) {
2396 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 2397 return kFALSE;
16a4353c 2398 }
348be253 2399 if (IsCheckCopy()) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 2400 if (!CheckInputData()) {
2401 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 2402 return kFALSE;
c57f56b7 2403 }
d3339be3 2404 if (!CreateDataset(fDataPattern)) {
2405 TString serror;
2406 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2407 if (fRunNumbers.Length()) serror = "run numbers";
2408 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2409 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2410 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2411 return kFALSE;
2412 }
c57f56b7 2413 WriteAnalysisFile();
2414 WriteAnalysisMacro();
2415 WriteExecutable();
2416 WriteValidationScript();
0f389141 2417 if (fMergeViaJDL) {
2418 WriteMergingMacro();
2419 WriteMergeExecutable();
2420 WriteValidationScript(kTRUE);
2421 }
5513444a 2422 if (!CreateJDL()) return kFALSE;
2423 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3bdcb562 2424 if (testMode) {
c57f56b7 2425 // Locally testing the analysis
2426 Info("StartAnalysis", "\n_______________________________________________________________________ \
2427 \n Running analysis script in a daughter shell as on a worker node \
2428 \n_______________________________________________________________________");
149d288c 2429 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 2430 TIter next(list);
2431 TObjString *str;
a2f5fc01 2432 TString outputFile;
c57f56b7 2433 while((str=(TObjString*)next())) {
a2f5fc01 2434 outputFile = str->GetString();
2435 Int_t index = outputFile.Index("@");
2436 if (index > 0) outputFile.Remove(index);
2437 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 2438 }
2439 delete list;
2440 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 2441 TString validationScript = fExecutable;
2442 validationScript.ReplaceAll(".sh", "_validation.sh");
2443 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 2444// gSystem->Exec("cat stdout");
5513444a 2445 return kFALSE;
c57f56b7 2446 }
5513444a 2447 // Check if submitting is managed by LPM manager
a3e84053 2448 if (fProductionMode) {
5513444a 2449 TString prodfile = fJDLName;
2450 prodfile.ReplaceAll(".jdl", ".prod");
2451 WriteProductionFile(prodfile);
2452 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2453 return kFALSE;
2454 }
a8739e8a 2455 // Submit AliEn job(s)
d2a409b2 2456 gGrid->Cd(fGridOutputDir);
a8739e8a 2457 TGridResult *res;
c57f56b7 2458 TString jobID = "";
d2a409b2 2459 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 2460 // Submit a given xml or a set of runs
a8739e8a 2461 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2462 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2463 if (res) {
2464 const char *cjobId = res->GetKey(0,"jobId");
2465 if (!cjobId) {
a03be957 2466 gGrid->Stdout();
2467 gGrid->Stderr();
a8739e8a 2468 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2469 return kFALSE;
a8739e8a 2470 } else {
2471 Info("StartAnalysis", "\n_______________________________________________________________________ \
2472 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2473 \n_______________________________________________________________________",
2474 fJDLName.Data(), cjobId);
2475 jobID = cjobId;
2476 }
2477 delete res;
a03be957 2478 } else {
2479 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2480 return kFALSE;
a8739e8a 2481 }
2482 } else {
d2a409b2 2483 // Submit for a range of enumeration of runs.
a03be957 2484 if (!Submit()) return kFALSE;
c57f56b7 2485 }
a8739e8a 2486
c57f56b7 2487 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2488 \n You may exit at any time and terminate the job later using the option <terminate> \
2489 \n ##################################################################################", jobID.Data());
bb885a9e 2490 gSystem->Exec("aliensh");
5513444a 2491 return kTRUE;
c57f56b7 2492}
2493
d2a409b2 2494//______________________________________________________________________________
a03be957 2495Bool_t AliAnalysisAlien::Submit()
d2a409b2 2496{
2497// Submit all master jobs.
2498 Int_t nmasterjobs = fInputFiles->GetEntries();
2499 Long_t tshoot = gSystem->Now();
a03be957 2500 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2501 while (fNsubmitted < nmasterjobs) {
2502 Long_t now = gSystem->Now();
2503 if ((now-tshoot)>30000) {
2504 tshoot = now;
a03be957 2505 if (!SubmitNext()) return kFALSE;
d2a409b2 2506 }
2507 }
a03be957 2508 return kTRUE;
d2a409b2 2509}
2510
0f389141 2511//______________________________________________________________________________
a03be957 2512Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2513{
2514// Submit all merging jobs.
2515 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2516 gGrid->Cd(fGridOutputDir);
2517 TString mergeJDLName = fExecutable;
2518 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2519 Int_t ntosubmit = fInputFiles->GetEntries();
0f389141 2520 for (Int_t i=0; i<ntosubmit; i++) {
f866cba5 2521 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2522 runOutDir.ReplaceAll(".xml", "");
7c2cd90a 2523 if (fOutputToRunNo) {
2524 // The output directory is the run number
2525 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2526 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2527 } else {
2528 // The output directory is the master number in 3 digits format
2529 printf("### Submitting merging job for master <%03d>\n", i);
2530 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2531 }
2532 // Check now the number of merging stages.
2b222f7c 2533 TObjArray *list = fOutputFiles.Tokenize(",");
2534 TIter next(list);
2535 TObjString *str;
2536 TString outputFile;
2537 while((str=(TObjString*)next())) {
2538 outputFile = str->GetString();
2539 Int_t index = outputFile.Index("@");
2540 if (index > 0) outputFile.Remove(index);
2541 if (!fMergeExcludes.Contains(outputFile)) break;
2542 }
2543 delete list;
7c2cd90a 2544 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, kTRUE, mergeJDLName);
2545 if (!done) return kFALSE;
0f389141 2546 }
a03be957 2547 if (!ntosubmit) return kTRUE;
0f389141 2548 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
2549 \n You may exit at any time and terminate the job later using the option <terminate> but disabling SetMergeViaJDL\
2550 \n ##################################################################################");
2551 gSystem->Exec("aliensh");
a03be957 2552 return kTRUE;
0f389141 2553}
2554
d2a409b2 2555//______________________________________________________________________________
a03be957 2556Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2557{
08d5b699 2558// Submit next bunch of master jobs if the queue is free. The first master job is
2559// submitted right away, while the next will not be unless the previous was split.
2560// The plugin will not submit new master jobs if there are more that 500 jobs in
2561// waiting phase.
d2a409b2 2562 static Bool_t iscalled = kFALSE;
2563 static Int_t firstmaster = 0;
2564 static Int_t lastmaster = 0;
2565 static Int_t npermaster = 0;
a03be957 2566 if (iscalled) return kTRUE;
d2a409b2 2567 iscalled = kTRUE;
2568 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2569 Int_t ntosubmit = 0;
2570 TGridResult *res;
2571 TString jobID = "";
3bdcb562 2572 Int_t nmasterjobs = fInputFiles->GetEntries();
2573 if (!fNsubmitted) {
2574 ntosubmit = 1;
2575 if (!IsUseSubmitPolicy()) {
2576 if (ntosubmit>5)
2577 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2578 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2579 ntosubmit = nmasterjobs;
2580 }
2581 } else {
d2a409b2 2582 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2583 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2584 // If last master not split, just return
a03be957 2585 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2586 // No more than 100 waiting jobs
08d5b699 2587 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2588 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
08d5b699 2589 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
7586eedc 2590 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2591 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2592 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2593 }
d2a409b2 2594 for (Int_t i=0; i<ntosubmit; i++) {
2595 // Submit for a range of enumeration of runs.
a03be957 2596 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2597 TString query;
cd11251e 2598 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2599 runOutDir.ReplaceAll(".xml", "");
2600 if (fOutputToRunNo)
2601 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2602 else
2603 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2604 printf("********* %s\n",query.Data());
d2a409b2 2605 res = gGrid->Command(query);
2606 if (res) {
98ca124f 2607 TString cjobId1 = res->GetKey(0,"jobId");
2608 if (!cjobId1.Length()) {
d2a409b2 2609 iscalled = kFALSE;
a03be957 2610 gGrid->Stdout();
2611 gGrid->Stderr();
2612 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2613 return kFALSE;
d2a409b2 2614 } else {
2615 Info("StartAnalysis", "\n_______________________________________________________________________ \
2616 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2617 \n_______________________________________________________________________",
98ca124f 2618 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2619 jobID += cjobId1;
2620 jobID += " ";
98ca124f 2621 lastmaster = cjobId1.Atoi();
d2a409b2 2622 if (!firstmaster) firstmaster = lastmaster;
2623 fNsubmitted++;
2624 }
2625 delete res;
a03be957 2626 } else {
2627 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2628 return kFALSE;
d2a409b2 2629 }
2630 }
2631 iscalled = kFALSE;
a03be957 2632 return kTRUE;
d2a409b2 2633}
2634
c57f56b7 2635//______________________________________________________________________________
2636void AliAnalysisAlien::WriteAnalysisFile()
2637{
f10e8481 2638// Write current analysis manager into the file <analysisFile>
2639 TString analysisFile = fExecutable;
2640 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2641 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2642 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2643 if (!mgr || !mgr->IsInitialized()) {
2644 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2645 return;
2646 }
2647 // Check analysis type
2648 TObject *handler;
2649 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2650 handler = (TObject*)mgr->GetInputEventHandler();
2651 if (handler) {
2652 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2653 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2654 }
2655 TDirectory *cdir = gDirectory;
f10e8481 2656 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2657 if (file) {
e16a394c 2658 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2659 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2660 // Unless merging makes no sense
2661 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2662 mgr->Write();
2663 delete file;
fe2d7fc2 2664 // Enable termination for local jobs
2665 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2666 }
2667 if (cdir) cdir->cd();
f10e8481 2668 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2669 }
2670 Bool_t copy = kTRUE;
2671 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2672 if (copy) {
2673 CdWork();
2674 TString workdir = gGrid->GetHomeDirectory();
2675 workdir += fGridWorkingDir;
f10e8481 2676 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
2677 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
2678 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 2679 }
2680}
2681
2682//______________________________________________________________________________
2683void AliAnalysisAlien::WriteAnalysisMacro()
2684{
2685// Write the analysis macro that will steer the analysis in grid mode.
2686 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2687 ofstream out;
2688 out.open(fAnalysisMacro.Data(), ios::out);
2689 if (!out.good()) {
2690 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2691 return;
2692 }
5b273635 2693 Bool_t hasSTEERBase = kFALSE;
2694 Bool_t hasESD = kFALSE;
2695 Bool_t hasAOD = kFALSE;
2696 Bool_t hasANALYSIS = kFALSE;
2697 Bool_t hasANALYSISalice = kFALSE;
2698 Bool_t hasCORRFW = kFALSE;
c57f56b7 2699 TString func = fAnalysisMacro;
2700 TString type = "ESD";
2701 TString comment = "// Analysis using ";
2702 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
2703 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
2704 type = "AOD";
2705 comment += "AOD";
2706 }
0df6ccf2 2707 if (type!="AOD" && fFriendChainName!="") {
2708 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
2709 return;
2710 }
c57f56b7 2711 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
2712 else comment += " data";
2713 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
2714 func.ReplaceAll(".C", "");
2715 out << "void " << func.Data() << "()" << endl;
2716 out << "{" << endl;
2717 out << comment.Data() << endl;
2718 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 2719 out << " TStopwatch timer;" << endl;
2720 out << " timer.Start();" << endl << endl;
13f0c00b 2721 // Change temp directory to current one
2722 out << "// Set temporary merging directory to current one" << endl;
2723 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
c57f56b7 2724 out << "// load base root libraries" << endl;
2725 out << " gSystem->Load(\"libTree\");" << endl;
2726 out << " gSystem->Load(\"libGeom\");" << endl;
2727 out << " gSystem->Load(\"libVMC\");" << endl;
2728 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 2729 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 2730 if (fAdditionalRootLibs.Length()) {
47a4137d 2731 // in principle libtree /lib geom libvmc etc. can go into this list, too
2732 out << "// Add aditional libraries" << endl;
2733 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2734 TIter next(list);
2735 TObjString *str;
2736 while((str=(TObjString*)next())) {
2737 if (str->GetString().Contains(".so"))
2738 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 2739 }
47a4137d 2740 if (list) delete list;
d5c6455a 2741 }
47a4137d 2742 out << "// include path" << endl;
2743 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2744 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
57377eb5 2745 out << "// Load analysis framework libraries" << endl;
652561d5 2746 TString setupPar = "AliAnalysisAlien::SetupPar";
4e5c5506 2747 if (!fPackages) {
4e5c5506 2748 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2749 out << " gSystem->Load(\"libESD\");" << endl;
2750 out << " gSystem->Load(\"libAOD\");" << endl;
2751 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 2752 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2753 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 2754 } else {
4e5c5506 2755 TIter next(fPackages);
2756 TObject *obj;
57377eb5 2757 TString pkgname;
57377eb5 2758 while ((obj=next())) {
2759 pkgname = obj->GetName();
4478e6f1 2760 if (pkgname == "STEERBase" ||
2761 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2762 if (pkgname == "ESD" ||
2763 pkgname == "ESD.par") hasESD = kTRUE;
2764 if (pkgname == "AOD" ||
2765 pkgname == "AOD.par") hasAOD = kTRUE;
2766 if (pkgname == "ANALYSIS" ||
2767 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2768 if (pkgname == "ANALYSISalice" ||
2769 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2770 if (pkgname == "CORRFW" ||
2771 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 2772 }
2773 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 2774 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2775 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 2776 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2777 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 2778 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2779 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 2780 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2781 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 2782 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2783 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 2784 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2785 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 2786 out << "// Compile other par packages" << endl;
2787 next.Reset();
fcc9bb6f 2788 while ((obj=next())) {
2789 pkgname = obj->GetName();
4478e6f1 2790 if (pkgname == "STEERBase" ||
2791 pkgname == "STEERBase.par" ||
2792 pkgname == "ESD" ||
2793 pkgname == "ESD.par" ||
2794 pkgname == "AOD" ||
2795 pkgname == "AOD.par" ||
2796 pkgname == "ANALYSIS" ||
2797 pkgname == "ANALYSIS.par" ||
2798 pkgname == "ANALYSISalice" ||
2799 pkgname == "ANALYSISalice.par" ||
2800 pkgname == "CORRFW" ||
2801 pkgname == "CORRFW.par") continue;
5b273635 2802 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 2803 }
4e5c5506 2804 }
6da75e0b 2805 if (fAdditionalLibs.Length()) {
2806 out << "// Add aditional AliRoot libraries" << endl;
2807 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2808 TIter next(list);
2809 TObjString *str;
2810 while((str=(TObjString*)next())) {
2811 if (str->GetString().Contains(".so"))
2812 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
652561d5 2813 if (str->GetString().Contains(".par"))
2814 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
6da75e0b 2815 }
2816 if (list) delete list;
2817 }
2818 out << endl;
c57f56b7 2819 out << "// analysis source to be compiled at runtime (if any)" << endl;
2820 if (fAnalysisSource.Length()) {
2821 TObjArray *list = fAnalysisSource.Tokenize(" ");
2822 TIter next(list);
2823 TObjString *str;
2824 while((str=(TObjString*)next())) {
2825 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2826 }
2827 if (list) delete list;
2828 }
2829 out << endl;
0f389141 2830 if (fFastReadOption) {
d3339be3 2831 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
2832 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 2833 out << "// fast xrootd reading enabled" << endl;
2834 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2835 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
2836 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 2837 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 2838 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 2839 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2840 }
c57f56b7 2841 out << "// connect to AliEn and make the chain" << endl;
2842 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2843 if (IsUsingTags()) {
2844 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
2845 } else {
45e980c4 2846 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
c57f56b7 2847 }
2848 out << "// read the analysis manager from file" << endl;
f10e8481 2849 TString analysisFile = fExecutable;
2850 analysisFile.ReplaceAll(".sh", ".root");
2851 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 2852 out << " if (!file) return;" << endl;
2853 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2854 out << " AliAnalysisManager *mgr = 0;" << endl;
2855 out << " TKey *key;" << endl;
2856 out << " while ((key=(TKey*)nextkey())) {" << endl;
2857 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2858 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2859 out << " };" << endl;
2860 out << " if (!mgr) {" << endl;
205b201f 2861 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
c57f56b7 2862 out << " return;" << endl;
2863 out << " }" << endl << endl;
2864 out << " mgr->PrintStatus();" << endl;
52b6a92b 2865 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 2866 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 2867 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 2868 } else {
2869 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 2870 }
2871 }
c57f56b7 2872 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 2873 out << " timer.Stop();" << endl;
2874 out << " timer.Print();" << endl;
c57f56b7 2875 out << "}" << endl << endl;
2876 if (IsUsingTags()) {
2877 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
2878 out << "{" << endl;
2879 out << "// Create a chain using tags from the xml file." << endl;
2880 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
2881 out << " if (!coll) {" << endl;
2882 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2883 out << " return NULL;" << endl;
2884 out << " }" << endl;
2885 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
2886 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
2887 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
2888 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
2889 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
2890 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
2891 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
2892 out << " // Check if the cuts configuration file was provided" << endl;
2893 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
2894 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
2895 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2896 out << " }" << endl;
0df6ccf2 2897 if (fFriendChainName=="") {
2898 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2899 } else {
2900 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
2901 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
2902 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
2903 }
c57f56b7 2904 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
2905 out << " chain->ls();" << endl;
2906 out << " return chain;" << endl;
fcc9bb6f 2907 out << "}" << endl << endl;
c57f56b7 2908 if (gSystem->AccessPathName("ConfigureCuts.C")) {
2909 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
2910 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
2911 msg += " AliLHCTagCuts *lhcCuts,\n";
2912 msg += " AliDetectorTagCuts *detCuts,\n";
2913 msg += " AliEventTagCuts *evCuts)";
23329835 2914 Info("WriteAnalysisMacro", "%s", msg.Data());
c57f56b7 2915 }
0df6ccf2 2916 }
2917 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 2918 out <<"//________________________________________________________________________________" << endl;
c57f56b7 2919 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
2920 out << "{" << endl;
2921 out << "// Create a chain using url's from xml file" << endl;
2922 out << " TString treename = type;" << endl;
2923 out << " treename.ToLower();" << endl;
2924 out << " treename += \"Tree\";" << endl;
e02fee64 2925 out << " printf(\"***************************************\\n\");" << endl;
2926 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
2927 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 2928 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
2929 out << " if (!coll) {" << endl;
2930 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2931 out << " return NULL;" << endl;
2932 out << " }" << endl;
2933 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 2934 if(fFriendChainName!="") {
2935 out << " TChain *chainFriend = new TChain(treename);" << endl;
2936 }
c57f56b7 2937 out << " coll->Reset();" << endl;
0df6ccf2 2938 out << " while (coll->Next()) {" << endl;
2939 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
2940 if(fFriendChainName!="") {
2941 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
2942 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2943 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2944 out << " chainFriend->Add(fileFriend.Data());" << endl;
2945 }
2946 out << " }" << endl;
c57f56b7 2947 out << " if (!chain->GetNtrees()) {" << endl;
2948 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
2949 out << " return NULL;" << endl;
2950 out << " }" << endl;
0df6ccf2 2951 if(fFriendChainName!="") {
2952 out << " chain->AddFriend(chainFriend);" << endl;
2953 }
c57f56b7 2954 out << " return chain;" << endl;
fcc9bb6f 2955 out << "}" << endl << endl;
c57f56b7 2956 }
5b273635 2957 if (hasANALYSISalice) {
2958 out <<"//________________________________________________________________________________" << endl;
2959 out << "Bool_t SetupPar(const char *package) {" << endl;
2960 out << "// Compile the package and set it up." << endl;
2961 out << " TString pkgdir = package;" << endl;
2962 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2963 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2964 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2965 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2966 out << " // Check for BUILD.sh and execute" << endl;
2967 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2968 out << " printf(\"*******************************\\n\");" << endl;
2969 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2970 out << " printf(\"*******************************\\n\");" << endl;
2971 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2972 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2973 out << " gSystem->ChangeDirectory(cdir);" << endl;
2974 out << " return kFALSE;" << endl;
2975 out << " }" << endl;
2976 out << " } else {" << endl;
2977 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2978 out << " gSystem->ChangeDirectory(cdir);" << endl;
2979 out << " return kFALSE;" << endl;
2980 out << " }" << endl;
2981 out << " // Check for SETUP.C and execute" << endl;
2982 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2983 out << " printf(\"*******************************\\n\");" << endl;
2984 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2985 out << " printf(\"*******************************\\n\");" << endl;
2986 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2987 out << " } else {" << endl;
2988 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2989 out << " gSystem->ChangeDirectory(cdir);" << endl;
2990 out << " return kFALSE;" << endl;
2991 out << " }" << endl;
2992 out << " // Restore original workdir" << endl;
2993 out << " gSystem->ChangeDirectory(cdir);" << endl;
2994 out << " return kTRUE;" << endl;
2995 out << "}" << endl;
2996 }
c57f56b7 2997 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2998 }
2999 Bool_t copy = kTRUE;
3000 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3001 if (copy) {
3002 CdWork();
3003 TString workdir = gGrid->GetHomeDirectory();
3004 workdir += fGridWorkingDir;
3005 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3006 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3007 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3008 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3009 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3010 }
3011 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3012 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3013 }
3014}
3015
0f389141 3016//______________________________________________________________________________
3017void AliAnalysisAlien::WriteMergingMacro()
3018{
3019// Write a macro to merge the outputs per master job.
3020 if (!fMergeViaJDL) return;
3021 if (!fOutputFiles.Length()) {
3022 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3023 return;
3024 }
3025 TString mergingMacro = fExecutable;
3026 mergingMacro.ReplaceAll(".sh","_merge.C");
3027 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3028 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3029 ofstream out;
3030 out.open(mergingMacro.Data(), ios::out);
3031 if (!out.good()) {
3032 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3033 return;
3034 }
5b273635 3035 Bool_t hasSTEERBase = kFALSE;
3036 Bool_t hasESD = kFALSE;
3037 Bool_t hasAOD = kFALSE;
3038 Bool_t hasANALYSIS = kFALSE;
3039 Bool_t hasANALYSISalice = kFALSE;
3040 Bool_t hasCORRFW = kFALSE;
0f389141 3041 TString func = mergingMacro;
3042 TString comment;
3043 func.ReplaceAll(".C", "");
7c2cd90a 3044 out << "void " << func.Data() << "(const char *dir, Int_t stage=0, Int_t ichunk=0)" << endl;
0f389141 3045 out << "{" << endl;
3046 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3047 out << " TStopwatch timer;" << endl;
3048 out << " timer.Start();" << endl << endl;
7c2cd90a 3049 if (!fExecutableCommand.Contains("aliroot")) {
3050 out << "// load base root libraries" << endl;
3051 out << " gSystem->Load(\"libTree\");" << endl;
3052 out << " gSystem->Load(\"libGeom\");" << endl;
3053 out << " gSystem->Load(\"libVMC\");" << endl;
3054 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3055 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3056 }
0f389141 3057 if (fAdditionalRootLibs.Length()) {
3058 // in principle libtree /lib geom libvmc etc. can go into this list, too
3059 out << "// Add aditional libraries" << endl;
3060 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3061 TIter next(list);
3062 TObjString *str;
3063 while((str=(TObjString*)next())) {
3064 if (str->GetString().Contains(".so"))
3065 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3066 }
3067 if (list) delete list;
3068 }
3069 out << "// include path" << endl;
3070 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3071 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
3072 out << "// Load analysis framework libraries" << endl;
3073 if (!fPackages) {
7c2cd90a 3074 if (!fExecutableCommand.Contains("aliroot")) {
3075 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3076 out << " gSystem->Load(\"libESD\");" << endl;
3077 out << " gSystem->Load(\"libAOD\");" << endl;
3078 }
0f389141 3079 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3080 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3081 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3082 } else {
3083 TIter next(fPackages);
3084 TObject *obj;
3085 TString pkgname;
5b273635 3086 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 3087 while ((obj=next())) {
3088 pkgname = obj->GetName();
3089 if (pkgname == "STEERBase" ||
3090 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3091 if (pkgname == "ESD" ||
3092 pkgname == "ESD.par") hasESD = kTRUE;
3093 if (pkgname == "AOD" ||
3094 pkgname == "AOD.par") hasAOD = kTRUE;
3095 if (pkgname == "ANALYSIS" ||
3096 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3097 if (pkgname == "ANALYSISalice" ||
3098 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3099 if (pkgname == "CORRFW" ||
3100 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3101 }
5b273635 3102 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 3103 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 3104 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 3105 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 3106 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 3107 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 3108 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 3109 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 3110 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 3111 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 3112 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 3113 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 3114 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 3115 out << "// Compile other par packages" << endl;
3116 next.Reset();
3117 while ((obj=next())) {
3118 pkgname = obj->GetName();
3119 if (pkgname == "STEERBase" ||
3120 pkgname == "STEERBase.par" ||
3121 pkgname == "ESD" ||
3122 pkgname == "ESD.par" ||
3123 pkgname == "AOD" ||
3124 pkgname == "AOD.par" ||
3125 pkgname == "ANALYSIS" ||
3126 pkgname == "ANALYSIS.par" ||
3127 pkgname == "ANALYSISalice" ||
3128 pkgname == "ANALYSISalice.par" ||
3129 pkgname == "CORRFW" ||
3130 pkgname == "CORRFW.par") continue;
5b273635 3131 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 3132 }
3133 }
3134 if (fAdditionalLibs.Length()) {
3135 out << "// Add aditional AliRoot libraries" << endl;
3136 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3137 TIter next(list);
3138 TObjString *str;
3139 while((str=(TObjString*)next())) {
3140 if (str->GetString().Contains(".so"))
3141 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3142 }
3143 if (list) delete list;
3144 }
3145 out << endl;
3146 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3147 if (fAnalysisSource.Length()) {
3148 TObjArray *list = fAnalysisSource.Tokenize(" ");
3149 TIter next(list);
3150 TObjString *str;
3151 while((str=(TObjString*)next())) {
3152 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3153 }
3154 if (list) delete list;
3155 }
149d288c 3156 out << endl;
3157
0f389141 3158 if (fFastReadOption) {
3159 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3160 out << "// fast xrootd reading enabled" << endl;
3161 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 3162 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
3163 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 3164 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 3165 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 3166 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 3167 }
3168 // Change temp directory to current one
3169 out << "// Set temporary merging directory to current one" << endl;
3170 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 3171 out << "// Connect to AliEn" << endl;
3172 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
0cdf65a8 3173 out << " Bool_t laststage = kFALSE;" << endl;
7c2cd90a 3174 out << " TString outputDir = dir;" << endl;
0f389141 3175 out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
3176 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
1f0d1ca2 3177 out << " mergeExcludes += \"" << AliAnalysisManager::GetAnalysisManager()->GetExtraFiles() << "\";" << endl;
58268c13 3178 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
0f389141 3179 out << " TIter *iter = new TIter(list);" << endl;
3180 out << " TObjString *str;" << endl;
a2f5fc01 3181 out << " TString outputFile;" << endl;
0f389141 3182 out << " Bool_t merged = kTRUE;" << endl;
3183 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 3184 out << " outputFile = str->GetString();" << endl;
7c2cd90a 3185 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
a2f5fc01 3186 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3187 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 3188 out << " // Skip already merged outputs" << endl;
a2f5fc01 3189 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3190 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 3191 out << " continue;" << endl;
3192 out << " }" << endl;
a2f5fc01 3193 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
7c2cd90a 3194 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage, ichunk);" << endl;
0f389141 3195 out << " if (!merged) {" << endl;
a2f5fc01 3196 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
7c2cd90a 3197 out << " return;" << endl;
0f389141 3198 out << " }" << endl;
7c2cd90a 3199 out << " // Check if this was the last stage. If yes, run terminate for the tasks." << endl;
3200 out << " if (!gSystem->AccessPathName(outputFile)) laststage = kTRUE;" << endl;
0f389141 3201 out << " }" << endl;
7c2cd90a 3202 out << " // all outputs merged, validate" << endl;
3203 out << " ofstream out;" << endl;
3204 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3205 out << " out.close();" << endl;
3206 out << " // read the analysis manager from file" << endl;
f866cba5 3207 TString analysisFile = fExecutable;
3208 analysisFile.ReplaceAll(".sh", ".root");
7c2cd90a 3209 out << " if (!laststage) return;" << endl;
f866cba5 3210 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
7c2cd90a 3211 out << " if (!file) return;" << endl;
f866cba5 3212 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3213 out << " AliAnalysisManager *mgr = 0;" << endl;
3214 out << " TKey *key;" << endl;
3215 out << " while ((key=(TKey*)nextkey())) {" << endl;
3216 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3217 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3218 out << " };" << endl;
3219 out << " if (!mgr) {" << endl;
3220 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3221 out << " return;" << endl;
3222 out << " }" << endl << endl;
b385fec0 3223 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 3224 out << " mgr->PrintStatus();" << endl;
3225 if (AliAnalysisManager::GetAnalysisManager()) {
3226 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3227 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3228 } else {
3229 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
3230 }
3231 }
3232 out << " mgr->StartAnalysis(\"gridterminate\");" << endl;
0f389141 3233 out << "}" << endl << endl;
5b273635 3234 if (hasANALYSISalice) {
3235 out <<"//________________________________________________________________________________" << endl;
3236 out << "Bool_t SetupPar(const char *package) {" << endl;
3237 out << "// Compile the package and set it up." << endl;
3238 out << " TString pkgdir = package;" << endl;
3239 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
3240 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
3241 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3242 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3243 out << " // Check for BUILD.sh and execute" << endl;
3244 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3245 out << " printf(\"*******************************\\n\");" << endl;
3246 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3247 out << " printf(\"*******************************\\n\");" << endl;
3248 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3249 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3250 out << " gSystem->ChangeDirectory(cdir);" << endl;
3251 out << " return kFALSE;" << endl;
3252 out << " }" << endl;
3253 out << " } else {" << endl;
3254 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3255 out << " gSystem->ChangeDirectory(cdir);" << endl;
3256 out << " return kFALSE;" << endl;
3257 out << " }" << endl;
3258 out << " // Check for SETUP.C and execute" << endl;
3259 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3260 out << " printf(\"*******************************\\n\");" << endl;
3261 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3262 out << " printf(\"*******************************\\n\");" << endl;
3263 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3264 out << " } else {" << endl;
3265 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3266 out << " gSystem->ChangeDirectory(cdir);" << endl;
3267 out << " return kFALSE;" << endl;
3268 out << " }" << endl;
3269 out << " // Restore original workdir" << endl;
3270 out << " gSystem->ChangeDirectory(cdir);" << endl;
3271 out << " return kTRUE;" << endl;
3272 out << "}" << endl;
3273 }
0f389141 3274 }
3275 Bool_t copy = kTRUE;
3276 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3277 if (copy) {
3278 CdWork();
3279 TString workdir = gGrid->GetHomeDirectory();
3280 workdir += fGridWorkingDir;
3281 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3282 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3283 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3284 }
3285}
3286
3287//______________________________________________________________________________
3288Bool_t AliAnalysisAlien::SetupPar(const char *package)
3289{
205b201f 3290// Compile the par file archive pointed by <package>. This must be present in the current directory.
0f389141 3291// Note that for loading the compiled library. The current directory should have precedence in
3292// LD_LIBRARY_PATH
3293 TString pkgdir = package;
3294 pkgdir.ReplaceAll(".par","");
3295 gSystem->Exec(Form("tar xvzf %s.par", pkgdir.Data()));
3296 TString cdir = gSystem->WorkingDirectory();
3297 gSystem->ChangeDirectory(pkgdir);
3298 // Check for BUILD.sh and execute
3299 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3300 printf("**************************************************\n");
3301 printf("*** Building PAR archive %s\n", package);
3302 printf("**************************************************\n");
3303 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3304 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3305 gSystem->ChangeDirectory(cdir);
3306 return kFALSE;
3307 }
3308 } else {
3309 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3310 gSystem->ChangeDirectory(cdir);
3311 return kFALSE;
3312 }
3313 // Check for SETUP.C and execute
3314 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3315 printf("**************************************************\n");
3316 printf("*** Setup PAR archive %s\n", package);
3317 printf("**************************************************\n");
3318 gROOT->Macro("PROOF-INF/SETUP.C");
3319 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3320 } else {
3321 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3322 gSystem->ChangeDirectory(cdir);
3323 return kFALSE;
3324 }
3325 // Restore original workdir
3326 gSystem->ChangeDirectory(cdir);
3327 return kTRUE;
3328}
3329
c57f56b7 3330//______________________________________________________________________________
3331void AliAnalysisAlien::WriteExecutable()
3332{
3333// Generate the alien executable script.
3334 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3335 ofstream out;
3336 out.open(fExecutable.Data(), ios::out);
3337 if (out.bad()) {
5513444a 3338 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 3339 return;
3340 }
3341 out << "#!/bin/bash" << endl;
c57f56b7 3342 out << "echo \"=========================================\"" << endl;
3343 out << "echo \"############## PATH : ##############\"" << endl;
3344 out << "echo $PATH" << endl;
3345 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3346 out << "echo $LD_LIBRARY_PATH" << endl;
3347 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3348 out << "echo $ROOTSYS" << endl;
3349 out << "echo \"############## which root : ##############\"" << endl;
3350 out << "which root" << endl;
3351 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3352 out << "echo $ALICE_ROOT" << endl;
3353 out << "echo \"############## which aliroot : ##############\"" << endl;
3354 out << "which aliroot" << endl;
9c5ddadc 3355 out << "echo \"############## system limits : ##############\"" << endl;
3356 out << "ulimit -a" << endl;
3357 out << "echo \"############## memory : ##############\"" << endl;
3358 out << "free -m" << endl;
c57f56b7 3359 out << "echo \"=========================================\"" << endl << endl;
74a53467 3360 // Make sure we can properly compile par files
3361 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 3362 out << fExecutableCommand << " ";
631c0b05 3363 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 3364 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3365 out << "echo \"############## memory after: ##############\"" << endl;
3366 out << "free -m" << endl;
c57f56b7 3367 }
3368 Bool_t copy = kTRUE;
3369 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3370 if (copy) {
3371 CdWork();
3372 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 3373 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3374 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
c57f56b7 3375 workdir += fGridWorkingDir;
3376 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3377 if (FileExists(executable)) gGrid->Rm(executable);
3378 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
3379 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3380 }
3381}
3382
0f389141 3383//______________________________________________________________________________
3384void AliAnalysisAlien::WriteMergeExecutable()
3385{
3386// Generate the alien executable script for the merging job.
3387 if (!fMergeViaJDL) return;
3388 TString mergeExec = fExecutable;
3389 mergeExec.ReplaceAll(".sh", "_merge.sh");
3390 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3391 ofstream out;
3392 out.open(mergeExec.Data(), ios::out);
3393 if (out.bad()) {
3394 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3395 return;
3396 }
3397 out << "#!/bin/bash" << endl;
3398 out << "echo \"=========================================\"" << endl;
3399 out << "echo \"############## PATH : ##############\"" << endl;
3400 out << "echo $PATH" << endl;
3401 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3402 out << "echo $LD_LIBRARY_PATH" << endl;
3403 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3404 out << "echo $ROOTSYS" << endl;
3405 out << "echo \"############## which root : ##############\"" << endl;
3406 out << "which root" << endl;
3407 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3408 out << "echo $ALICE_ROOT" << endl;
3409 out << "echo \"############## which aliroot : ##############\"" << endl;
3410 out << "which aliroot" << endl;
3411 out << "echo \"############## system limits : ##############\"" << endl;
3412 out << "ulimit -a" << endl;
3413 out << "echo \"############## memory : ##############\"" << endl;
3414 out << "free -m" << endl;
3415 out << "echo \"=========================================\"" << endl << endl;
3416 // Make sure we can properly compile par files
3417 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
3418 TString mergeMacro = fExecutable;
3419 mergeMacro.ReplaceAll(".sh", "_merge.C");
7c2cd90a 3420 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2,$3)\"" << endl;
0f389141 3421 out << fExecutableCommand << " " << "$ARG" << endl;
3422 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3423 out << "echo \"############## memory after: ##############\"" << endl;
3424 out << "free -m" << endl;
0f389141 3425 }
3426 Bool_t copy = kTRUE;
3427 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3428 if (copy) {
3429 CdWork();
3430 TString workdir = gGrid->GetHomeDirectory();
3431 TString bindir = Form("%s/bin", workdir.Data());
b93f8109 3432 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
0f389141 3433 workdir += fGridWorkingDir;
3434 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3435 if (FileExists(executable)) gGrid->Rm(executable);
3436 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
3437 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3438 }
3439}
3440
c57f56b7 3441//______________________________________________________________________________
5513444a 3442void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3443{
3444// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 3445// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 3446// Next lines: full_path_to_dataset XXX (XXX is a string)
3447// To submit, one has to: submit jdl XXX for all lines
3448 ofstream out;
3449 out.open(filename, ios::out);
3450 if (out.bad()) {
3451 Error("WriteProductionFile", "Bad file name: %s", filename);
3452 return;
3453 }
3454 TString workdir = gGrid->GetHomeDirectory();
3455 workdir += fGridWorkingDir;
f5e8c702 3456 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 3457 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 3458 out << locjdl << " " << njobspermaster << endl;
5513444a 3459 Int_t nmasterjobs = fInputFiles->GetEntries();
3460 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 3461 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3462 runOutDir.ReplaceAll(".xml", "");
3463 if (fOutputToRunNo)
3464 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3465 else
3466 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 3467 }
3468 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
fdbbc7be 3469 if (FileExists(filename)) gGrid->Rm(filename);
5513444a 3470 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3471}
3472
3473//______________________________________________________________________________
0f389141 3474void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3475{
3476// Generate the alien validation script.
3477 // Generate the validation script
3478 TObjString *os;
0d5d317c 3479 TString validationScript = fExecutable;
0f389141 3480 if (merge) validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
3481 else validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 3482 if (!Connect()) {
3483 Error("WriteValidationScript", "Alien connection required");
3484 return;
3485 }
a2f5fc01 3486 TString outStream = "";
3487 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3488 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3489 ofstream out;
0d5d317c 3490 out.open(validationScript, ios::out);
c57f56b7 3491 out << "#!/bin/bash" << endl;
3492 out << "##################################################" << endl;
3493 out << "validateout=`dirname $0`" << endl;
3494 out << "validatetime=`date`" << endl;
3495 out << "validated=\"0\";" << endl;
3496 out << "error=0" << endl;
3497 out << "if [ -z $validateout ]" << endl;
3498 out << "then" << endl;
3499 out << " validateout=\".\"" << endl;
3500 out << "fi" << endl << endl;
3501 out << "cd $validateout;" << endl;
3502 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3503 out << "echo \"*******************************************************\"" << outStream << endl;
3504 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3505 out << "" << endl;
a2f5fc01 3506 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3507 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3508 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3509 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3510 out << "ls -la ./" << outStream << endl;
3511 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3512 out << "##################################################" << endl;
ebec370a 3513 out << "" << endl;
3514
3515 out << "if [ ! -f stderr ] ; then" << endl;
3516 out << " error=1" << endl;
a2f5fc01 3517 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3518 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3519 out << "fi" << endl;
3520
b34c9f51 3521 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3522 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3523 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3524 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3525 out << "" << endl;
3526
ebec370a 3527 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3528 out << " error=1" << endl;
a2f5fc01 3529 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3530 out << " echo \"$parArch\" " << outStream << endl;
3531 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3532 out << "fi" << endl;
3533
3534 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3535 out << " error=1" << endl;
a2f5fc01 3536 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3537 out << " echo \"$segViol\" " << outStream << endl;
3538 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3539 out << "fi" << endl;
3540
3541 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3542 out << " error=1" << endl;
a2f5fc01 3543 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3544 out << " echo \"$segFault\" " << outStream << endl;
3545 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3546 out << "fi" << endl;
3547
b34c9f51 3548 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3549 out << " error=1" << endl;
a2f5fc01 3550 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3551 out << " echo \"$glibcErr\" " << outStream << endl;
3552 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3553 out << "fi" << endl;
3554
ebec370a 3555 // Part dedicated to the specific analyses running into the train
3556
149d288c 3557 TObjArray *arr = fOutputFiles.Tokenize(",");
c57f56b7 3558 TIter next1(arr);
a2f5fc01 3559 TString outputFile;
5b9b4998 3560 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3561 TString extra = mgr->GetExtraFiles();
c57f56b7 3562 while ((os=(TObjString*)next1())) {
7c2cd90a 3563 if (merge) break;
a2f5fc01 3564 outputFile = os->GetString();
3565 Int_t index = outputFile.Index("@");
3566 if (index > 0) outputFile.Remove(index);
3567 if (merge && fMergeExcludes.Contains(outputFile)) continue;
3568 if (extra.Contains(outputFile)) continue;
3569 if (outputFile.Contains("*")) continue;
3570 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3571 out << " error=1" << endl;
7c2cd90a 3572 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3573 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
c57f56b7 3574 out << "fi" << endl;
3575 }
3576 delete arr;
7c2cd90a 3577 out << "if ! [ -f outputs_valid ] ; then" << endl;
3578 out << " error=1" << endl;
3579 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3580 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3581 out << "fi" << endl;
923e2ca5 3582
c57f56b7 3583 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3584 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3585 if (!IsKeepLogs()) {
3586 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3587 outStream = "";
149d288c 3588 out << " rm -f std*" << endl;
3589 }
c57f56b7 3590 out << "fi" << endl;
3591
a2f5fc01 3592 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3593 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3594 out << "cd -" << endl;
3595 out << "exit $error" << endl;
3596 }
3597 Bool_t copy = kTRUE;
3598 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3599 if (copy) {
3600 CdWork();
3601 TString workdir = gGrid->GetHomeDirectory();
3602 workdir += fGridWorkingDir;
0d5d317c 3603 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
3604 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3605 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3606 }
3607}