]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
take care of gGeoManager handling
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
25#include "TError.h"
c57f56b7 26#include "TROOT.h"
27#include "TSystem.h"
28#include "TFile.h"
29#include "TObjString.h"
30#include "TObjArray.h"
31#include "TGrid.h"
32#include "TGridResult.h"
33#include "TGridCollection.h"
34#include "TGridJDL.h"
d2a409b2 35#include "TGridJobStatusList.h"
36#include "TGridJobStatus.h"
c57f56b7 37#include "TFileMerger.h"
38#include "AliAnalysisManager.h"
bb885a9e 39#include "AliVEventHandler.h"
40#include "AliAnalysisDataContainer.h"
c57f56b7 41#include "AliAnalysisAlien.h"
42
43ClassImp(AliAnalysisAlien)
44
45//______________________________________________________________________________
46AliAnalysisAlien::AliAnalysisAlien()
47 :AliAnalysisGrid(),
48 fGridJDL(NULL),
0f389141 49 fMergingJDL(NULL),
c57f56b7 50 fPrice(0),
51 fTTL(0),
52 fSplitMaxInputFileNumber(0),
53 fMaxInitFailed(0),
54 fMasterResubmitThreshold(0),
bb885a9e 55 fNtestFiles(0),
319593fb 56 fNrunsPerMaster(0),
16a4353c 57 fMaxMergeFiles(0),
d2a409b2 58 fNsubmitted(0),
a3e84053 59 fProductionMode(0),
cd11251e 60 fOutputToRunNo(0),
0f389141 61 fMergeViaJDL(0),
62 fFastReadOption(0),
e1c22e21 63 fOverwriteMode(1),
149d288c 64 fNreplicas(2),
c57f56b7 65 fRunNumbers(),
66 fExecutable(),
0a1c1f7f 67 fExecutableCommand(),
c57f56b7 68 fArguments(),
631c0b05 69 fExecutableArgs(),
c57f56b7 70 fAnalysisMacro(),
71 fAnalysisSource(),
d5c6455a 72 fAdditionalRootLibs(),
c57f56b7 73 fAdditionalLibs(),
74 fSplitMode(),
75 fAPIVersion(),
76 fROOTVersion(),
77 fAliROOTVersion(),
648174cf 78 fExternalPackages(),
c57f56b7 79 fUser(),
80 fGridWorkingDir(),
81 fGridDataDir(),
82 fDataPattern(),
83 fGridOutputDir(),
84 fOutputArchive(),
85 fOutputFiles(),
86 fInputFormat(),
e7c71df0 87 fDatasetName(),
c57f56b7 88 fJDLName(),
bb885a9e 89 fMergeExcludes(),
f965131e 90 fIncludePath(),
bb885a9e 91 fCloseSE(),
0df6ccf2 92 fFriendChainName(),
c6cb3634 93 fJobTag(),
648174cf 94 fOutputSingle(),
5fce53f4 95 fRunPrefix(),
4e5c5506 96 fInputFiles(0),
97 fPackages(0)
c57f56b7 98{
99// Dummy ctor.
100 SetDefaults();
101}
102
103//______________________________________________________________________________
104AliAnalysisAlien::AliAnalysisAlien(const char *name)
105 :AliAnalysisGrid(name),
106 fGridJDL(NULL),
0f389141 107 fMergingJDL(NULL),
c57f56b7 108 fPrice(0),
109 fTTL(0),
110 fSplitMaxInputFileNumber(0),
111 fMaxInitFailed(0),
112 fMasterResubmitThreshold(0),
bb885a9e 113 fNtestFiles(0),
319593fb 114 fNrunsPerMaster(0),
16a4353c 115 fMaxMergeFiles(0),
d2a409b2 116 fNsubmitted(0),
a3e84053 117 fProductionMode(0),
cd11251e 118 fOutputToRunNo(0),
0f389141 119 fMergeViaJDL(0),
120 fFastReadOption(0),
e1c22e21 121 fOverwriteMode(1),
149d288c 122 fNreplicas(2),
c57f56b7 123 fRunNumbers(),
124 fExecutable(),
0a1c1f7f 125 fExecutableCommand(),
c57f56b7 126 fArguments(),
631c0b05 127 fExecutableArgs(),
c57f56b7 128 fAnalysisMacro(),
129 fAnalysisSource(),
d5c6455a 130 fAdditionalRootLibs(),
c57f56b7 131 fAdditionalLibs(),
132 fSplitMode(),
133 fAPIVersion(),
134 fROOTVersion(),
135 fAliROOTVersion(),
648174cf 136 fExternalPackages(),
c57f56b7 137 fUser(),
138 fGridWorkingDir(),
139 fGridDataDir(),
140 fDataPattern(),
141 fGridOutputDir(),
142 fOutputArchive(),
143 fOutputFiles(),
144 fInputFormat(),
e7c71df0 145 fDatasetName(),
c57f56b7 146 fJDLName(),
bb885a9e 147 fMergeExcludes(),
f965131e 148 fIncludePath(),
bb885a9e 149 fCloseSE(),
0df6ccf2 150 fFriendChainName(),
c6cb3634 151 fJobTag(),
648174cf 152 fOutputSingle(),
5fce53f4 153 fRunPrefix(),
4e5c5506 154 fInputFiles(0),
155 fPackages(0)
c57f56b7 156{
157// Default ctor.
158 SetDefaults();
159}
160
161//______________________________________________________________________________
162AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
163 :AliAnalysisGrid(other),
164 fGridJDL(NULL),
0f389141 165 fMergingJDL(NULL),
c57f56b7 166 fPrice(other.fPrice),
167 fTTL(other.fTTL),
168 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
169 fMaxInitFailed(other.fMaxInitFailed),
170 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 171 fNtestFiles(other.fNtestFiles),
319593fb 172 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 173 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 174 fNsubmitted(other.fNsubmitted),
a3e84053 175 fProductionMode(other.fProductionMode),
cd11251e 176 fOutputToRunNo(other.fOutputToRunNo),
0f389141 177 fMergeViaJDL(other.fMergeViaJDL),
178 fFastReadOption(other.fFastReadOption),
179 fOverwriteMode(other.fOverwriteMode),
149d288c 180 fNreplicas(other.fNreplicas),
c57f56b7 181 fRunNumbers(other.fRunNumbers),
182 fExecutable(other.fExecutable),
0a1c1f7f 183 fExecutableCommand(other.fExecutableCommand),
c57f56b7 184 fArguments(other.fArguments),
631c0b05 185 fExecutableArgs(other.fExecutableArgs),
c57f56b7 186 fAnalysisMacro(other.fAnalysisMacro),
187 fAnalysisSource(other.fAnalysisSource),
d5c6455a 188 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 189 fAdditionalLibs(other.fAdditionalLibs),
190 fSplitMode(other.fSplitMode),
191 fAPIVersion(other.fAPIVersion),
192 fROOTVersion(other.fROOTVersion),
193 fAliROOTVersion(other.fAliROOTVersion),
648174cf 194 fExternalPackages(other.fExternalPackages),
c57f56b7 195 fUser(other.fUser),
196 fGridWorkingDir(other.fGridWorkingDir),
197 fGridDataDir(other.fGridDataDir),
198 fDataPattern(other.fDataPattern),
199 fGridOutputDir(other.fGridOutputDir),
200 fOutputArchive(other.fOutputArchive),
201 fOutputFiles(other.fOutputFiles),
202 fInputFormat(other.fInputFormat),
e7c71df0 203 fDatasetName(other.fDatasetName),
c57f56b7 204 fJDLName(other.fJDLName),
bb885a9e 205 fMergeExcludes(other.fMergeExcludes),
f965131e 206 fIncludePath(other.fIncludePath),
bb885a9e 207 fCloseSE(other.fCloseSE),
0df6ccf2 208 fFriendChainName(other.fFriendChainName),
c6cb3634 209 fJobTag(other.fJobTag),
648174cf 210 fOutputSingle(other.fOutputSingle),
5fce53f4 211 fRunPrefix(other.fRunPrefix),
4e5c5506 212 fInputFiles(0),
213 fPackages(0)
c57f56b7 214{
215// Copy ctor.
216 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 217 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 218 fRunRange[0] = other.fRunRange[0];
219 fRunRange[1] = other.fRunRange[1];
c57f56b7 220 if (other.fInputFiles) {
221 fInputFiles = new TObjArray();
222 TIter next(other.fInputFiles);
223 TObject *obj;
224 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
225 fInputFiles->SetOwner();
226 }
4e5c5506 227 if (other.fPackages) {
228 fPackages = new TObjArray();
229 TIter next(other.fPackages);
230 TObject *obj;
231 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
232 fPackages->SetOwner();
233 }
c57f56b7 234}
235
236//______________________________________________________________________________
237AliAnalysisAlien::~AliAnalysisAlien()
238{
239// Destructor.
240 if (fGridJDL) delete fGridJDL;
0f389141 241 if (fMergingJDL) delete fMergingJDL;
c57f56b7 242 if (fInputFiles) delete fInputFiles;
4e5c5506 243 if (fPackages) delete fPackages;
c57f56b7 244}
245
246//______________________________________________________________________________
247AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
248{
249// Assignment.
250 if (this != &other) {
251 AliAnalysisGrid::operator=(other);
252 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 253 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 254 fPrice = other.fPrice;
255 fTTL = other.fTTL;
256 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
257 fMaxInitFailed = other.fMaxInitFailed;
258 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 259 fNtestFiles = other.fNtestFiles;
a3e84053 260 fNrunsPerMaster = other.fNrunsPerMaster;
261 fMaxMergeFiles = other.fMaxMergeFiles;
262 fNsubmitted = other.fNsubmitted;
263 fProductionMode = other.fProductionMode;
cd11251e 264 fOutputToRunNo = other.fOutputToRunNo;
0f389141 265 fMergeViaJDL = other.fMergeViaJDL;
266 fFastReadOption = other.fFastReadOption;
267 fOverwriteMode = other.fOverwriteMode;
149d288c 268 fNreplicas = other.fNreplicas;
c57f56b7 269 fRunNumbers = other.fRunNumbers;
270 fExecutable = other.fExecutable;
0a1c1f7f 271 fExecutableCommand = other.fExecutableCommand;
c57f56b7 272 fArguments = other.fArguments;
631c0b05 273 fExecutableArgs = other.fExecutableArgs;
c57f56b7 274 fAnalysisMacro = other.fAnalysisMacro;
275 fAnalysisSource = other.fAnalysisSource;
d5c6455a 276 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 277 fAdditionalLibs = other.fAdditionalLibs;
278 fSplitMode = other.fSplitMode;
279 fAPIVersion = other.fAPIVersion;
280 fROOTVersion = other.fROOTVersion;
281 fAliROOTVersion = other.fAliROOTVersion;
648174cf 282 fExternalPackages = other.fExternalPackages;
c57f56b7 283 fUser = other.fUser;
284 fGridWorkingDir = other.fGridWorkingDir;
285 fGridDataDir = other.fGridDataDir;
286 fDataPattern = other.fDataPattern;
287 fGridOutputDir = other.fGridOutputDir;
288 fOutputArchive = other.fOutputArchive;
289 fOutputFiles = other.fOutputFiles;
290 fInputFormat = other.fInputFormat;
e7c71df0 291 fDatasetName = other.fDatasetName;
c57f56b7 292 fJDLName = other.fJDLName;
bb885a9e 293 fMergeExcludes = other.fMergeExcludes;
f965131e 294 fIncludePath = other.fIncludePath;
bb885a9e 295 fCloseSE = other.fCloseSE;
0df6ccf2 296 fFriendChainName = other.fFriendChainName;
c6cb3634 297 fJobTag = other.fJobTag;
648174cf 298 fOutputSingle = other.fOutputSingle;
5fce53f4 299 fRunPrefix = other.fRunPrefix;
c57f56b7 300 if (other.fInputFiles) {
301 fInputFiles = new TObjArray();
302 TIter next(other.fInputFiles);
303 TObject *obj;
304 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
305 fInputFiles->SetOwner();
306 }
4e5c5506 307 if (other.fPackages) {
308 fPackages = new TObjArray();
309 TIter next(other.fPackages);
310 TObject *obj;
311 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
312 fPackages->SetOwner();
313 }
c57f56b7 314 }
315 return *this;
316}
317
f965131e 318//______________________________________________________________________________
319void AliAnalysisAlien::AddIncludePath(const char *path)
320{
321// Add include path in the remote analysis macro.
322 TString p(path);
323 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
324 else fIncludePath += Form("-I%s ", path);
325}
326
c57f56b7 327//______________________________________________________________________________
328void AliAnalysisAlien::AddRunNumber(Int_t run)
329{
330// Add a run number to the list of runs to be processed.
331 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 332 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 333}
334
ee75cfc3 335//______________________________________________________________________________
336void AliAnalysisAlien::AddRunNumber(const char* run)
337{
338// Add a run number to the list of runs to be processed.
339 if (fRunNumbers.Length()) fRunNumbers += " ";
340 fRunNumbers += run;
341}
342
c57f56b7 343//______________________________________________________________________________
344void AliAnalysisAlien::AddDataFile(const char *lfn)
345{
346// Adds a data file to the input to be analysed. The file should be a valid LFN
347// or point to an existing file in the alien workdir.
348 if (!fInputFiles) fInputFiles = new TObjArray();
349 fInputFiles->Add(new TObjString(lfn));
350}
648174cf 351
352//______________________________________________________________________________
353void AliAnalysisAlien::AddExternalPackage(const char *package)
354{
355// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
356 if (fExternalPackages) fExternalPackages += " ";
357 fExternalPackages += package;
358}
359
c57f56b7 360//______________________________________________________________________________
361Bool_t AliAnalysisAlien::Connect()
362{
363// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
364 if (gGrid && gGrid->IsConnected()) return kTRUE;
c57f56b7 365 if (!gGrid) {
366 Info("Connect", "Trying to connect to AliEn ...");
367 TGrid::Connect("alien://");
368 }
369 if (!gGrid || !gGrid->IsConnected()) {
370 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
371 return kFALSE;
372 }
373 fUser = gGrid->GetUser();
374 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
375 return kTRUE;
376}
377
378//______________________________________________________________________________
379void AliAnalysisAlien::CdWork()
380{
381// Check validity of alien workspace. Create directory if possible.
382 if (!Connect()) {
383 Error("CdWork", "Alien connection required");
384 return;
385 }
386 TString homedir = gGrid->GetHomeDirectory();
387 TString workdir = homedir + fGridWorkingDir;
923e2ca5 388 if (DirectoryExists(workdir)) {
389 gGrid->Cd(workdir);
390 return;
391 }
392 // Work directory not existing - create it
393 gGrid->Cd(homedir);
394 if (gGrid->Mkdir(workdir)) {
395 gGrid->Cd(fGridWorkingDir);
396 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
397 } else {
398 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
399 workdir.Data(), homedir.Data());
400 fGridWorkingDir = "";
401 }
c57f56b7 402}
403
348be253 404//______________________________________________________________________________
405Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
406{
407// Check if file copying is possible.
408 if (!Connect()) {
409 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
410 return kFALSE;
411 }
d3339be3 412 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
413 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 414 // Check if alien_CLOSE_SE is defined
415 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
416 if (!closeSE.IsNull()) {
417 Info("CheckFileCopy", "Your current close storage is pointing to: \
418 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
419 } else {
420 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
421 }
422 // Check if grid directory exists.
423 if (!DirectoryExists(alienpath)) {
424 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
425 return kFALSE;
426 }
d3339be3 427 TFile f("plugin_test_copy", "RECREATE");
348be253 428 // User may not have write permissions to current directory
429 if (f.IsZombie()) {
430 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
431 gSystem->WorkingDirectory());
432 return kFALSE;
433 }
434 f.Close();
d3339be3 435 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
436 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
437 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 438 \n# 1. Make sure you have write permissions there. If this is the case: \
439 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
440 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
441 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
442 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
443 gSystem->Unlink(f.GetName());
444 return kFALSE;
445 }
446 gSystem->Unlink(f.GetName());
d3339be3 447 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
448 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 449 return kTRUE;
450}
451
c57f56b7 452//______________________________________________________________________________
453Bool_t AliAnalysisAlien::CheckInputData()
454{
455// Check validity of input data. If necessary, create xml files.
a8739e8a 456 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 457 if (!fGridDataDir.Length()) {
458 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
459 return kFALSE;
460 }
461 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
462 return kTRUE;
c57f56b7 463 }
464 // Process declared files
a2f5fc01 465 Bool_t isCollection = kFALSE;
466 Bool_t isXml = kFALSE;
467 Bool_t useTags = kFALSE;
c57f56b7 468 Bool_t checked = kFALSE;
469 CdWork();
470 TString file;
471 TString workdir = gGrid->GetHomeDirectory();
472 workdir += fGridWorkingDir;
473 if (fInputFiles) {
474 TObjString *objstr;
475 TIter next(fInputFiles);
476 while ((objstr=(TObjString*)next())) {
477 file = workdir;
478 file += "/";
479 file += objstr->GetString();
480 // Store full lfn path
481 if (FileExists(file)) objstr->SetString(file);
482 else {
483 file = objstr->GetName();
484 if (!FileExists(objstr->GetName())) {
485 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
486 objstr->GetName(), workdir.Data());
487 return kFALSE;
488 }
489 }
490 Bool_t iscoll, isxml, usetags;
491 CheckDataType(file, iscoll, isxml, usetags);
492 if (!checked) {
493 checked = kTRUE;
a2f5fc01 494 isCollection = iscoll;
495 isXml = isxml;
496 useTags = usetags;
497 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 498 } else {
a2f5fc01 499 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
c57f56b7 500 Error("CheckInputData", "Some conflict was found in the types of inputs");
501 return kFALSE;
502 }
503 }
504 }
505 }
506 // Process requested run numbers
a8739e8a 507 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 508 // Check validity of alien data directory
509 if (!fGridDataDir.Length()) {
510 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
511 return kFALSE;
512 }
923e2ca5 513 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 514 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
515 return kFALSE;
516 }
a2f5fc01 517 if (isCollection) {
c57f56b7 518 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
519 return kFALSE;
520 }
521
a2f5fc01 522 if (checked && !isXml) {
c57f56b7 523 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
524 return kFALSE;
525 }
526 // Check validity of run number(s)
527 TObjArray *arr;
528 TObjString *os;
319593fb 529 Int_t nruns = 0;
904f9f5f 530 TString schunk, schunk2;
c57f56b7 531 TString path;
532 if (!checked) {
533 checked = kTRUE;
a2f5fc01 534 useTags = fDataPattern.Contains("tag");
535 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
c57f56b7 536 }
a2f5fc01 537 if (useTags != fDataPattern.Contains("tag")) {
c57f56b7 538 Error("CheckInputData", "Cannot mix input files using/not using tags");
539 return kFALSE;
540 }
541 if (fRunNumbers.Length()) {
a8739e8a 542 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 543 arr = fRunNumbers.Tokenize(" ");
544 TIter next(arr);
545 while ((os=(TObjString*)next())) {
546 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 547 if (!DirectoryExists(path)) {
548 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 549 continue;
c57f56b7 550 }
551 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
552 TString msg = "\n##### file: ";
553 msg += path;
554 msg += " type: xml_collection;";
a2f5fc01 555 if (useTags) msg += " using_tags: Yes";
c57f56b7 556 else msg += " using_tags: No";
557 Info("CheckDataType", msg.Data());
319593fb 558 if (fNrunsPerMaster<2) {
d2a409b2 559 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 560 } else {
561 nruns++;
562 if (((nruns-1)%fNrunsPerMaster) == 0) {
563 schunk = os->GetString();
564 }
565 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
566 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 567 AddDataFile(schunk);
319593fb 568 }
c57f56b7 569 }
570 delete arr;
a8739e8a 571 } else {
572 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
573 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 574 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 575 if (!DirectoryExists(path)) {
576// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 577 continue;
578 }
5fce53f4 579 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 580 TString msg = "\n##### file: ";
581 msg += path;
582 msg += " type: xml_collection;";
a2f5fc01 583 if (useTags) msg += " using_tags: Yes";
a8739e8a 584 else msg += " using_tags: No";
585 Info("CheckDataType", msg.Data());
319593fb 586 if (fNrunsPerMaster<2) {
5fce53f4 587 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 588 } else {
589 nruns++;
590 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 591 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 592 }
904f9f5f 593 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 594 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 595 schunk += schunk2;
d2a409b2 596 AddDataFile(schunk);
319593fb 597 }
a8739e8a 598 }
904f9f5f 599 if (!fInputFiles) {
600 schunk += schunk2;
601 AddDataFile(schunk);
602 }
c57f56b7 603 }
604 return kTRUE;
605}
606
607//______________________________________________________________________________
608Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
609{
610// Create dataset for the grid data directory + run number.
f9e1936d 611 if (TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
c57f56b7 612 if (!Connect()) {
613 Error("CreateDataset", "Cannot create dataset with no grid connection");
614 return kFALSE;
615 }
616
617 // Cd workspace
618 CdWork();
619 TString workdir = gGrid->GetHomeDirectory();
620 workdir += fGridWorkingDir;
621
622 // Compose the 'find' command arguments
623 TString command;
624 TString options = "-x collection ";
bb885a9e 625 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 626 TString conditions = "";
627
628 TString file;
629 TString path;
319593fb 630 Int_t nruns = 0;
904f9f5f 631 TString schunk, schunk2;
ab254fd1 632 TGridCollection *cbase=0, *cadd=0;
d2a409b2 633 if (!fRunNumbers.Length() && !fRunRange[0]) {
634 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
635 // Make a single data collection from data directory.
636 path = fGridDataDir;
923e2ca5 637 if (!DirectoryExists(path)) {
d2a409b2 638 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
639 return kFALSE;
640 }
923e2ca5 641// CdWork();
d2a409b2 642 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
643 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 644 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 645 command = "find ";
646 command += options;
647 command += path;
648 command += " ";
649 command += pattern;
650 command += conditions;
84fcd93f 651 printf("command: %s\n", command.Data());
d2a409b2 652 TGridResult *res = gGrid->Command(command);
653 if (res) delete res;
654 // Write standard output to file
655 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 656 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
657 Bool_t nullFile = kFALSE;
658 if (!hasGrep) {
d3339be3 659 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
660 } else {
a2f5fc01 661 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
662 if (nullFile) {
defd7a3a 663 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
664 return kFALSE;
665 }
d3339be3 666 }
0f389141 667 }
668 Bool_t fileExists = FileExists(file);
669 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 670 // Copy xml file to alien space
0f389141 671 if (fileExists) gGrid->Rm(file);
d2a409b2 672 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
673 if (!FileExists(file)) {
674 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
675 return kFALSE;
676 }
677 // Update list of files to be processed.
678 }
679 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
680 return kTRUE;
681 }
c57f56b7 682 // Several runs
a2f5fc01 683 Bool_t nullResult = kTRUE;
a8739e8a 684 if (fRunNumbers.Length()) {
685 TObjArray *arr = fRunNumbers.Tokenize(" ");
686 TObjString *os;
687 TIter next(arr);
688 while ((os=(TObjString*)next())) {
689 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 690 if (!DirectoryExists(path)) continue;
691// CdWork();
a8739e8a 692 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
693 else file = Form("%s.xml", os->GetString().Data());
319593fb 694 // If local collection file does not exist, create it via 'find' command.
d3339be3 695 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 696 command = "find ";
697 command += options;
698 command += path;
699 command += pattern;
700 command += conditions;
701 TGridResult *res = gGrid->Command(command);
702 if (res) delete res;
703 // Write standard output to file
704 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 705 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
706 Bool_t nullFile = kFALSE;
707 if (!hasGrep) {
d3339be3 708 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
709 } else {
a2f5fc01 710 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
711 if (nullFile) {
defd7a3a 712 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
713 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
714 continue;
715 }
d3339be3 716 }
a2f5fc01 717 nullResult = kFALSE;
d3339be3 718 }
a8739e8a 719 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 720 // Check if there is one run per master job.
721 if (fNrunsPerMaster<2) {
722 if (FileExists(file)) {
0f389141 723 if (fOverwriteMode) gGrid->Rm(file);
724 else {
725 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
726 continue;
727 }
319593fb 728 }
729 // Copy xml file to alien space
730 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
731 if (!FileExists(file)) {
732 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
733 delete arr;
734 return kFALSE;
735 }
736 } else {
737 nruns++;
738 if (((nruns-1)%fNrunsPerMaster) == 0) {
739 schunk = os->GetString();
740 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
741 } else {
742 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 743 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 744 cbase->Add(cadd);
745 delete cadd;
746 }
747 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
748 continue;
749 }
750 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 751 if (FileExists(schunk)) {
752 if (fOverwriteMode) gGrid->Rm(file);
753 else {
754 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
755 continue;
756 }
319593fb 757 }
84fcd93f 758 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 759 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 760 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 761 if (!FileExists(schunk)) {
762 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
763 delete arr;
764 return kFALSE;
765 }
d3339be3 766 }
a8739e8a 767 }
768 delete arr;
a2f5fc01 769 if (nullResult) {
d3339be3 770 Error("CreateDataset", "No valid dataset corresponding to the query!");
771 return kFALSE;
772 }
a8739e8a 773 } else {
774 // Process a full run range.
775 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 776 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 777 if (!DirectoryExists(path)) continue;
778// CdWork();
a8739e8a 779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 780 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
0f389141 781 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
782 if (fOverwriteMode) gGrid->Rm(file);
783 else {
784 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
785 continue;
786 }
a8739e8a 787 }
319593fb 788 // If local collection file does not exist, create it via 'find' command.
d3339be3 789 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 790 command = "find ";
791 command += options;
792 command += path;
793 command += pattern;
794 command += conditions;
795 TGridResult *res = gGrid->Command(command);
796 if (res) delete res;
797 // Write standard output to file
798 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
a2f5fc01 799 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
800 Bool_t nullFile = kFALSE;
801 if (!hasGrep) {
d3339be3 802 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
803 } else {
a2f5fc01 804 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
805 if (nullFile) {
defd7a3a 806 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
807 continue;
808 }
d3339be3 809 }
a2f5fc01 810 nullResult = kFALSE;
319593fb 811 }
a8739e8a 812 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 813 // Check if there is one run per master job.
814 if (fNrunsPerMaster<2) {
815 if (FileExists(file)) {
0f389141 816 if (fOverwriteMode) gGrid->Rm(file);
817 else {
818 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
819 continue;
820 }
319593fb 821 }
822 // Copy xml file to alien space
823 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
824 if (!FileExists(file)) {
825 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
826 return kFALSE;
827 }
828 } else {
829 nruns++;
95e5b448 830 // Check if the collection for the chunk exist locally.
831 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 832 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
833 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
834 else continue;
835 }
84fcd93f 836 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 837 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 838 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 839 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
840 } else {
841 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
842 cbase->Add(cadd);
843 delete cadd;
844 }
904f9f5f 845 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
846 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 847 continue;
848 }
904f9f5f 849 schunk = schunk2;
319593fb 850 if (FileExists(schunk)) {
0f389141 851 if (fOverwriteMode) gGrid->Rm(schunk);
852 else {
853 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
854 continue;
855 }
319593fb 856 }
84fcd93f 857 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 858 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 859 if (FileExists(schunk)) {
0f389141 860 if (fOverwriteMode) gGrid->Rm(schunk);
861 else {
862 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
863 continue;
864 }
95e5b448 865 }
319593fb 866 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
867 if (!FileExists(schunk)) {
868 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
869 return kFALSE;
870 }
871 }
c57f56b7 872 }
a2f5fc01 873 if (nullResult) {
d3339be3 874 Error("CreateDataset", "No valid dataset corresponding to the query!");
875 return kFALSE;
876 }
a8739e8a 877 }
c57f56b7 878 return kTRUE;
879}
880
881//______________________________________________________________________________
882Bool_t AliAnalysisAlien::CreateJDL()
883{
884// Generate a JDL file according to current settings. The name of the file is
885// specified by fJDLName.
886 Bool_t error = kFALSE;
887 TObjArray *arr = 0;
888 Bool_t copy = kTRUE;
889 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
890 Bool_t generate = kTRUE;
891 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
892 if (!Connect()) {
893 Error("CreateJDL", "Alien connection required");
894 return kFALSE;
895 }
896 // Check validity of alien workspace
897 CdWork();
898 TString workdir = gGrid->GetHomeDirectory();
899 workdir += fGridWorkingDir;
900 if (generate) {
901 TObjString *os;
902 if (!fInputFiles) {
903 Error("CreateJDL()", "Define some input files for your analysis.");
904 error = kTRUE;
905 }
906 // Compose list of input files
907 // Check if output files were defined
908 if (!fOutputFiles.Length()) {
909 Error("CreateJDL", "You must define at least one output file");
910 error = kTRUE;
911 }
912 // Check if an output directory was defined and valid
913 if (!fGridOutputDir.Length()) {
914 Error("CreateJDL", "You must define AliEn output directory");
915 error = kTRUE;
916 } else {
d2a409b2 917 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 918 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 919 if (gGrid->Mkdir(fGridOutputDir)) {
920 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
921 } else {
922 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 923 // error = kTRUE;
c57f56b7 924 }
925 }
926 gGrid->Cd(workdir);
927 }
928 // Exit if any error up to now
929 if (error) return kFALSE;
930 // Set JDL fields
0f389141 931 if (!fUser.IsNull()) {
932 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
933 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
934 }
935 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
936 TString mergeExec = fExecutable;
937 mergeExec.ReplaceAll(".sh", "_merge.sh");
938 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
939 mergeExec.ReplaceAll(".sh", ".C");
940 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 941 if (!fArguments.IsNull())
942 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
1f0d1ca2 943 fMergingJDL->SetArguments("$1");
944 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
945 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
946 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
947 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
948
0f389141 949 if (fMaxInitFailed > 0) {
c57f56b7 950 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 951 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
952 }
953 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 954 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 955 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
956 }
957 if (fSplitMode.Length()) {
c57f56b7 958 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 959 fGridJDL->SetDescription("Split", "We split per SE or file");
960 }
961 if (!fAliROOTVersion.IsNull()) {
962 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
963 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
964 }
965 if (!fROOTVersion.IsNull()) {
c57f56b7 966 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 967 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
968 }
969 if (!fAPIVersion.IsNull()) {
c57f56b7 970 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 971 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
972 }
648174cf 973 if (!fExternalPackages.IsNull()) {
974 arr = fExternalPackages.Tokenize(" ");
975 TIter next(arr);
976 while ((os=(TObjString*)next())) {
977 TString pkgname = os->GetString();
978 Int_t index = pkgname.Index("::");
979 TString pkgversion = pkgname(index+2, pkgname.Length());
980 pkgname.Remove(index);
981 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 982 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 983 }
984 delete arr;
985 }
0f389141 986 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
987 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
988 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 989 TString analysisFile = fExecutable;
990 analysisFile.ReplaceAll(".sh", ".root");
991 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 992 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 993 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
994 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
995 if (fAdditionalLibs.Length()) {
996 arr = fAdditionalLibs.Tokenize(" ");
997 TIter next(arr);
998 while ((os=(TObjString*)next())) {
999 if (os->GetString().Contains(".so")) continue;
1000 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 1001 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 1002 }
1003 delete arr;
1004 }
4e5c5506 1005 if (fPackages) {
1006 TIter next(fPackages);
1007 TObject *obj;
0f389141 1008 while ((obj=next())) {
4e5c5506 1009 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1010 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1011 }
4e5c5506 1012 }
c57f56b7 1013 if (fOutputArchive.Length()) {
1014 arr = fOutputArchive.Tokenize(" ");
1015 TIter next(arr);
0f389141 1016 Bool_t first = kTRUE;
1017 const char *comment = "Files to be archived";
1018 const char *comment1 = comment;
1019 while ((os=(TObjString*)next())) {
1020 if (!first) comment = NULL;
1021 if (!os->GetString().Contains("@") && fCloseSE.Length())
1022 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1023 else
1024 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1025 first = kFALSE;
1026 }
c57f56b7 1027 delete arr;
0f389141 1028 TString outputArchive = fOutputArchive;
1029 if (!fMergeExcludes.IsNull()) {
1030 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1031 TIter next1(arr);
1032 while ((os=(TObjString*)next1())) {
0f389141 1033 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1034 outputArchive.ReplaceAll(os->GetString(),"");
1035 }
1036 delete arr;
1037 }
1038 arr = outputArchive.Tokenize(" ");
705adb3e 1039 TIter next2(arr);
0f389141 1040 comment = comment1;
1041 first = kTRUE;
705adb3e 1042 while ((os=(TObjString*)next2())) {
0f389141 1043 if (!first) comment = NULL;
1044 if (!os->GetString().Contains("@") && fCloseSE.Length())
1045 fMergingJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1046 else
1047 fMergingJDL->AddToOutputArchive(os->GetString(), comment);
1048 first = kFALSE;
1049 }
1050 delete arr;
c57f56b7 1051 }
149d288c 1052 arr = fOutputFiles.Tokenize(",");
c57f56b7 1053 TIter next(arr);
0f389141 1054 Bool_t first = kTRUE;
1055 const char *comment = "Files to be archived";
1056 const char *comment1 = comment;
43da816a 1057 while ((os=(TObjString*)next())) {
e1eaf596 1058 // Ignore ouputs in jdl that are also in outputarchive
1059 TString sout = os->GetString();
1060 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1061 if (fOutputArchive.Contains(sout)) continue;
0f389141 1062 if (!first) comment = NULL;
43da816a 1063 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1064 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1065 else
0f389141 1066 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
1067 first = kFALSE;
43da816a 1068 }
c57f56b7 1069 delete arr;
0f389141 1070 if (fOutputFiles.Length()) {
1071 TString outputFiles = fOutputFiles;
1072 if (!fMergeExcludes.IsNull()) {
1073 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1074 TIter next1(arr);
1075 while ((os=(TObjString*)next1())) {
0f389141 1076 outputFiles.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1077 outputFiles.ReplaceAll(os->GetString(),"");
1078 }
1079 delete arr;
1080 }
1081 arr = outputFiles.Tokenize(" ");
705adb3e 1082 TIter next2(arr);
0f389141 1083 comment = comment1;
1084 first = kTRUE;
705adb3e 1085 while ((os=(TObjString*)next2())) {
0f389141 1086 // Ignore ouputs in jdl that are also in outputarchive
1087 TString sout = os->GetString();
1088 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1089 if (fOutputArchive.Contains(sout)) continue;
1090 if (!first) comment = NULL;
1091 if (!os->GetString().Contains("@") && fCloseSE.Length())
1092 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1093 else
1094 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
1095 }
1096 delete arr;
1097 }
1098 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1099 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
0d5d317c 1100 TString validationScript = fExecutable;
1101 validationScript.ReplaceAll(".sh", "_validation.sh");
0f389141 1102 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1103 validationScript = fExecutable;
1104 validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
1105 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1106 if (fMasterResubmitThreshold) {
1107 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1108 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1109 }
d2a409b2 1110 // Write a jdl with 2 input parameters: collection name and output dir name.
1111 WriteJDL(copy);
c57f56b7 1112 }
1113 // Copy jdl to grid workspace
a8739e8a 1114 if (copy) {
b5e4aaa7 1115 // Check if an output directory was defined and valid
1116 if (!fGridOutputDir.Length()) {
1117 Error("CreateJDL", "You must define AliEn output directory");
1118 return kFALSE;
1119 } else {
1120 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1121 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b5e4aaa7 1122 if (gGrid->Mkdir(fGridOutputDir)) {
1123 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1124 } else {
1125 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1126 return kFALSE;
1127 }
1128 }
1129 gGrid->Cd(workdir);
1130 }
648174cf 1131 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1132 TString mergeJDLName = fExecutable;
1133 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1134 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1135 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1136 if (fProductionMode) {
648174cf 1137 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1138 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1139 }
648174cf 1140 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1141 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1142 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1143 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1144 if (fMergeViaJDL) {
1145 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1146 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1147 }
648174cf 1148 }
c57f56b7 1149 if (fAdditionalLibs.Length()) {
e7c71df0 1150 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1151 TObjString *os;
1152 TIter next(arr);
1153 while ((os=(TObjString*)next())) {
c57f56b7 1154 if (os->GetString().Contains(".so")) continue;
4e5c5506 1155 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1156 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1157 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1158 }
1159 delete arr;
1160 }
4e5c5506 1161 if (fPackages) {
1162 TIter next(fPackages);
1163 TObject *obj;
1164 while ((obj=next())) {
fdbbc7be 1165 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1166 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1167 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1168 }
1169 }
c57f56b7 1170 }
1171 return kTRUE;
1172}
1173
a8739e8a 1174//______________________________________________________________________________
d2a409b2 1175Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1176{
1177// Writes one or more JDL's corresponding to findex. If findex is negative,
1178// all run numbers are considered in one go (jdl). For non-negative indices
1179// they correspond to the indices in the array fInputFiles.
1180 if (!fInputFiles) return kFALSE;
1181 TObjString *os;
a8739e8a 1182 TString workdir = gGrid->GetHomeDirectory();
1183 workdir += fGridWorkingDir;
d2a409b2 1184
1185 if (!fRunNumbers.Length() && !fRunRange[0]) {
1186 // One jdl with no parameters in case input data is specified by name.
a8739e8a 1187 TIter next(fInputFiles);
1188 while ((os=(TObjString*)next()))
0f389141 1189 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()), "Input xml collections");
648174cf 1190 if (!fOutputSingle.IsNull())
0f389141 1191 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1192 else {
1193 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1194 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1195 }
a8739e8a 1196 } else {
d2a409b2 1197 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
0f389141 1198 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
cd11251e 1199 if (!fOutputSingle.IsNull()) {
0f389141 1200 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1201 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
cd11251e 1202 } else {
0f389141 1203 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1204 fMergingJDL->SetOutputDirectory(Form("%s/$1", fGridOutputDir.Data()), "Output directory");
cd11251e 1205 }
a8739e8a 1206 }
1207
1208
1209 // Generate the JDL as a string
1210 TString sjdl = fGridJDL->Generate();
0f389141 1211 TString sjdl1 = fMergingJDL->Generate();
a8739e8a 1212 Int_t index;
a8739e8a 1213 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1214 sjdl.ReplaceAll("(member", "\n (member");
1215 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1216 sjdl.ReplaceAll("{", "{\n ");
1217 sjdl.ReplaceAll("};", "\n};");
1218 sjdl.ReplaceAll("{\n \n", "{\n");
1219 sjdl.ReplaceAll("\n\n", "\n");
1220 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1221 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1222 sjdl1.ReplaceAll("(member", "\n (member");
1223 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1224 sjdl1.ReplaceAll("{", "{\n ");
1225 sjdl1.ReplaceAll("};", "\n};");
1226 sjdl1.ReplaceAll("{\n \n", "{\n");
1227 sjdl1.ReplaceAll("\n\n", "\n");
1228 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1229 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1230 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1231 index = sjdl.Index("JDLVariables");
1232 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
0f389141 1233 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1234 sjdl1.Prepend(Form("Jobtag = {\n \"comment:Merging_%s\"\n};\n", fJobTag.Data()));
1235 index = sjdl1.Index("JDLVariables");
1236 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
a8739e8a 1237 // Write jdl to file
a8739e8a 1238 ofstream out;
d2a409b2 1239 out.open(fJDLName.Data(), ios::out);
a8739e8a 1240 if (out.bad()) {
d2a409b2 1241 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1242 return kFALSE;
1243 }
1244 out << sjdl << endl;
0f389141 1245 TString mergeJDLName = fExecutable;
1246 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1247 if (fMergeViaJDL) {
1248 ofstream out1;
1249 out1.open(mergeJDLName.Data(), ios::out);
1250 if (out.bad()) {
1251 Error("CreateJDL", "Bad file name: %s", mergeJDLName.Data());
1252 return kFALSE;
1253 }
1254 out1 << sjdl1 << endl;
1255 }
a8739e8a 1256
1257 // Copy jdl to grid workspace
1258 if (!copy) {
1259 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1260 } else {
d2a409b2 1261 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1262 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1263 if (fProductionMode) {
b5fe9cba 1264 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1265 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1266 }
d2a409b2 1267 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1268 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1269 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1270 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1271 if (fMergeViaJDL) {
1272 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1273 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1274 }
a8739e8a 1275 }
1276 return kTRUE;
1277}
1278
c57f56b7 1279//______________________________________________________________________________
5513444a 1280Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1281{
1282// Returns true if file exists.
5513444a 1283 if (!gGrid) return kFALSE;
c57f56b7 1284 TGridResult *res = gGrid->Ls(lfn);
1285 if (!res) return kFALSE;
1286 TMap *map = dynamic_cast<TMap*>(res->At(0));
1287 if (!map) {
1288 delete res;
1289 return kFALSE;
1290 }
1291 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1292 if (!objs || !objs->GetString().Length()) {
1293 delete res;
1294 return kFALSE;
1295 }
1296 delete res;
1297 return kTRUE;
1298}
1299
923e2ca5 1300//______________________________________________________________________________
1301Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1302{
1303// Returns true if directory exists. Can be also a path.
1304 if (!gGrid) return kFALSE;
1305 // Check if dirname is a path
1306 TString dirstripped = dirname;
1307 dirstripped = dirstripped.Strip();
1308 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1309 TString dir = gSystem->BaseName(dirstripped);
1310 dir += "/";
1311 TString path = gSystem->DirName(dirstripped);
1312 TGridResult *res = gGrid->Ls(path, "-F");
1313 if (!res) return kFALSE;
1314 TIter next(res);
1315 TMap *map;
1316 TObject *obj;
1317 while ((map=dynamic_cast<TMap*>(next()))) {
1318 obj = map->GetValue("name");
1319 if (!obj) break;
1320 if (dir == obj->GetName()) {
1321 delete res;
1322 return kTRUE;
1323 }
1324 }
1325 delete res;
1326 return kFALSE;
1327}
1328
c57f56b7 1329//______________________________________________________________________________
a2f5fc01 1330void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
c57f56b7 1331{
1332// Check input data type.
a2f5fc01 1333 isCollection = kFALSE;
1334 isXml = kFALSE;
1335 useTags = kFALSE;
c57f56b7 1336 if (!gGrid) {
1337 Error("CheckDataType", "No connection to grid");
1338 return;
1339 }
a2f5fc01 1340 isCollection = IsCollection(lfn);
c57f56b7 1341 TString msg = "\n##### file: ";
1342 msg += lfn;
a2f5fc01 1343 if (isCollection) {
c57f56b7 1344 msg += " type: raw_collection;";
1345 // special treatment for collections
a2f5fc01 1346 isXml = kFALSE;
c57f56b7 1347 // check for tag files in the collection
1348 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1349 if (!res) {
1350 msg += " using_tags: No (unknown)";
1351 Info("CheckDataType", msg.Data());
1352 return;
1353 }
1354 const char* typeStr = res->GetKey(0, "origLFN");
1355 if (!typeStr || !strlen(typeStr)) {
1356 msg += " using_tags: No (unknown)";
1357 Info("CheckDataType", msg.Data());
1358 return;
1359 }
1360 TString file = typeStr;
a2f5fc01 1361 useTags = file.Contains(".tag");
1362 if (useTags) msg += " using_tags: Yes";
c57f56b7 1363 else msg += " using_tags: No";
1364 Info("CheckDataType", msg.Data());
1365 return;
1366 }
1367 TString slfn(lfn);
1368 slfn.ToLower();
a2f5fc01 1369 isXml = slfn.Contains(".xml");
1370 if (isXml) {
c57f56b7 1371 // Open xml collection and check if there are tag files inside
1372 msg += " type: xml_collection;";
1373 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1374 if (!coll) {
1375 msg += " using_tags: No (unknown)";
1376 Info("CheckDataType", msg.Data());
1377 return;
1378 }
1379 TMap *map = coll->Next();
1380 if (!map) {
1381 msg += " using_tags: No (unknown)";
1382 Info("CheckDataType", msg.Data());
1383 return;
1384 }
1385 map = (TMap*)map->GetValue("");
1386 TString file;
1387 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
a2f5fc01 1388 useTags = file.Contains(".tag");
c57f56b7 1389 delete coll;
a2f5fc01 1390 if (useTags) msg += " using_tags: Yes";
c57f56b7 1391 else msg += " using_tags: No";
1392 Info("CheckDataType", msg.Data());
1393 return;
1394 }
a2f5fc01 1395 useTags = slfn.Contains(".tag");
c57f56b7 1396 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1397 else msg += " type: unknown file;";
a2f5fc01 1398 if (useTags) msg += " using_tags: Yes";
c57f56b7 1399 else msg += " using_tags: No";
1400 Info("CheckDataType", msg.Data());
1401}
1402
4e5c5506 1403//______________________________________________________________________________
1404void AliAnalysisAlien::EnablePackage(const char *package)
1405{
1406// Enables a par file supposed to exist in the current directory.
1407 TString pkg(package);
1408 pkg.ReplaceAll(".par", "");
1409 pkg += ".par";
1410 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1411 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1412 return;
1413 }
1414 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1415 Info("EnablePackage", "AliEn plugin will use .par packages");
1416 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1417 if (!fPackages) {
1418 fPackages = new TObjArray();
1419 fPackages->SetOwner();
1420 }
1421 fPackages->Add(new TObjString(pkg));
1422}
1423
d2a409b2 1424//______________________________________________________________________________
1425const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1426{
1427// Get job status for all jobs with jobid>jobidstart.
1428 static char mstatus[20];
1429 mstatus[0] = '\0';
1430 nrunning = 0;
1431 nwaiting = 0;
1432 nerror = 0;
1433 ndone = 0;
1434 TGridJobStatusList *list = gGrid->Ps("");
1435 if (!list) return mstatus;
1436 Int_t nentries = list->GetSize();
1437 TGridJobStatus *status;
1438 Int_t pid;
1439 for (Int_t ijob=0; ijob<nentries; ijob++) {
1440 status = (TGridJobStatus *)list->At(ijob);
1441 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1442 if (pid<jobidstart) continue;
1443 if (pid == lastid) {
1444 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1445 }
1446 switch (status->GetStatus()) {
1447 case TGridJobStatus::kWAITING:
1448 nwaiting++; break;
1449 case TGridJobStatus::kRUNNING:
1450 nrunning++; break;
1451 case TGridJobStatus::kABORTED:
1452 case TGridJobStatus::kFAIL:
1453 case TGridJobStatus::kUNKNOWN:
1454 nerror++; break;
1455 case TGridJobStatus::kDONE:
1456 ndone++;
1457 }
1458 }
1459 list->Delete();
1460 delete list;
1461 return mstatus;
1462}
1463
c57f56b7 1464//______________________________________________________________________________
1465Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1466{
1467// Returns true if file is a collection. Functionality duplicated from
1468// TAlien::Type() because we don't want to directly depend on TAlien.
1469 if (!gGrid) {
1470 Error("IsCollection", "No connection to grid");
1471 return kFALSE;
1472 }
1473 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1474 if (!res) return kFALSE;
1475 const char* typeStr = res->GetKey(0, "type");
1476 if (!typeStr || !strlen(typeStr)) return kFALSE;
1477 if (!strcmp(typeStr, "collection")) return kTRUE;
1478 delete res;
1479 return kFALSE;
1480}
1481
fe2d7fc2 1482//______________________________________________________________________________
1483Bool_t AliAnalysisAlien::IsSingleOutput() const
1484{
1485// Check if single-ouput option is on.
1486 return (!fOutputSingle.IsNull());
1487}
1488
16a4353c 1489//______________________________________________________________________________
1490void AliAnalysisAlien::Print(Option_t *) const
1491{
1492// Print current plugin settings.
84fcd93f 1493 printf("### AliEn analysis plugin current settings ###\n");
e1c22e21 1494 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1495 if (fOverwriteMode) {
1496 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1497 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1498 }
348be253 1499 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1500 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1501 printf("= Production mode:______________________________ %d\n", fProductionMode);
1502 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1503 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1504 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1505 if (fUser.Length())
84fcd93f 1506 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1507 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1508 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1509 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1510 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1511 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1512 if (fRunNumbers.Length())
84fcd93f 1513 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1514 if (fRunRange[0])
5fce53f4 1515 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1516 if (!fRunRange[0] && !fRunNumbers.Length()) {
1517 TIter next(fInputFiles);
1518 TObject *obj;
1519 TString list;
1520 while ((obj=next())) list += obj->GetName();
84fcd93f 1521 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1522 }
1523 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1524 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1525 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1526 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1527 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1528 printf("=====================================================================\n");
1529 printf("= Job price: ___________________________________ %d\n", fPrice);
1530 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1531 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1532 if (fMaxInitFailed>0)
84fcd93f 1533 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1534 if (fMasterResubmitThreshold>0)
84fcd93f 1535 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
149d288c 1536 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
319593fb 1537 if (fNrunsPerMaster>0)
84fcd93f 1538 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1539 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1540 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1541 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1542 if (fArguments.Length())
84fcd93f 1543 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1544 if (fExecutableArgs.Length())
1545 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1546 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1547 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1548 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1549 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1550 if (fDatasetName)
84fcd93f 1551 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1552 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1553 if (fIncludePath.Data())
84fcd93f 1554 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1555 if (fCloseSE.Length())
84fcd93f 1556 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1557 if (fFriendChainName.Length())
84fcd93f 1558 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1559 if (fPackages) {
1560 TIter next(fPackages);
1561 TObject *obj;
1562 TString list;
1563 while ((obj=next())) list += obj->GetName();
84fcd93f 1564 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1565 }
1566}
1567
c57f56b7 1568//______________________________________________________________________________
1569void AliAnalysisAlien::SetDefaults()
1570{
1571// Set default values for everything. What cannot be filled will be left empty.
1572 if (fGridJDL) delete fGridJDL;
1573 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1574 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1575 fPrice = 1;
1576 fTTL = 30000;
1577 fSplitMaxInputFileNumber = 100;
1578 fMaxInitFailed = 0;
1579 fMasterResubmitThreshold = 0;
bb885a9e 1580 fNtestFiles = 10;
149d288c 1581 fNreplicas = 2;
a8739e8a 1582 fRunRange[0] = 0;
1583 fRunRange[1] = 0;
319593fb 1584 fNrunsPerMaster = 1;
16a4353c 1585 fMaxMergeFiles = 100;
c57f56b7 1586 fRunNumbers = "";
1587 fExecutable = "analysis.sh";
0a1c1f7f 1588 fExecutableCommand = "root -b -q";
c57f56b7 1589 fArguments = "";
631c0b05 1590 fExecutableArgs = "";
c57f56b7 1591 fAnalysisMacro = "myAnalysis.C";
1592 fAnalysisSource = "";
1593 fAdditionalLibs = "";
1594 fSplitMode = "se";
1595 fAPIVersion = "";
1596 fROOTVersion = "";
1597 fAliROOTVersion = "";
1598 fUser = ""; // Your alien user name
1599 fGridWorkingDir = "";
1600 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1601 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1602 fFriendChainName = "";
c57f56b7 1603 fGridOutputDir = "output";
149d288c 1604 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
c57f56b7 1605 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1606 fInputFormat = "xml-single";
1607 fJDLName = "analysis.jdl";
c6cb3634 1608 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1609 fMergeExcludes = "";
0f389141 1610 fMergeViaJDL = 0;
348be253 1611 SetUseCopy(kTRUE);
1612 SetCheckCopy(kTRUE);
149d288c 1613 SetDefaultOutputs(kTRUE);
e1c22e21 1614 fOverwriteMode = 1;
c57f56b7 1615}
1616
0f389141 1617//______________________________________________________________________________
1618Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge)
1619{
1620// Merge all registered outputs from basedir.
a2f5fc01 1621 TString outputFile = output;
0f389141 1622 TString command;
a2f5fc01 1623 TString outputChunk;
1624 TString previousChunk = "";
1625 Int_t countChunk = 0;
1626 Int_t countZero = nmaxmerge;
0f389141 1627 Bool_t merged = kTRUE;
a2f5fc01 1628 Int_t index = outputFile.Index("@");
1629 if (index > 0) outputFile.Remove(index);
1630 command = Form("find %s/ *%s", basedir, outputFile.Data());
0f389141 1631 printf("command: %s\n", command.Data());
1632 TGridResult *res = gGrid->Command(command);
1633 if (!res) {
1634 printf("Error: No result for the find command\n");
1635 return kFALSE;
1636 }
1637
1638 TFileMerger *fm = 0;
1639 TIter nextmap(res);
1640 TMap *map = 0;
1641 // Check if there is a merge operation to resume
a2f5fc01 1642 outputChunk = outputFile;
1643 outputChunk.ReplaceAll(".root", "_*.root");
0f389141 1644 // Check for existent temporary merge files
e1c22e21 1645 // Check overwrite mode and remove previous partial results if needed
a2f5fc01 1646 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
0f389141 1647 while (1) {
1648 // Skip as many input files as in a chunk
1649 for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
1650 if (!map) {
a2f5fc01 1651 ::Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", outputFile.Data(), res->GetSize());
0f389141 1652 delete res;
1653 return kFALSE;
1654 }
a2f5fc01 1655 outputChunk = outputFile;
1656 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
1657 countChunk++;
1658 if (gSystem->AccessPathName(outputChunk)) continue;
1659 // Merged file with chunks up to <countChunk> found
1660 printf("Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
1661 previousChunk = outputChunk;
0f389141 1662 break;
1663 }
1664 }
a2f5fc01 1665 countZero = nmaxmerge;
0f389141 1666
1667 while ((map=(TMap*)nextmap())) {
1668 // Loop 'find' results and get next LFN
a2f5fc01 1669 if (countZero == nmaxmerge) {
0f389141 1670 // First file in chunk - create file merger and add previous chunk if any.
1671 fm = new TFileMerger(kFALSE);
1672 fm->SetFastMethod(kTRUE);
a2f5fc01 1673 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
1674 outputChunk = outputFile;
1675 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
0f389141 1676 }
1677 // If last file found, put merged results in the output file
a2f5fc01 1678 if (map == res->Last()) outputChunk = outputFile;
0f389141 1679 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1680 if (!objs || !objs->GetString().Length()) {
1681 // Nothing found - skip this output
1682 delete res;
1683 delete fm;
1684 return kFALSE;
1685 }
1686 // Add file to be merged and decrement chunk counter.
1687 fm->AddFile(objs->GetString());
a2f5fc01 1688 countZero--;
1689 if (countZero==0 || map == res->Last()) {
1690 fm->OutputFile(outputChunk);
0f389141 1691 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1692 // Nothing found - skip this output
a2f5fc01 1693 ::Warning("MergeOutputs", "No <%s> files found.", outputFile.Data());
0f389141 1694 delete res;
1695 delete fm;
1696 return kFALSE;
1697 }
1698 // Merge the outputs, then go to next chunk
1699 if (!fm->Merge()) {
a2f5fc01 1700 ::Error("MergeOutputs", "Could not merge all <%s> files", outputFile.Data());
0f389141 1701 delete res;
1702 delete fm;
1703 merged = kFALSE;
1704 return kFALSE;
1705 } else {
a2f5fc01 1706 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
1707 gSystem->Unlink(previousChunk);
0f389141 1708 }
1709 if (map == res->Last()) {
1710 delete res;
1711 delete fm;
1712 break;
1713 }
a2f5fc01 1714 countChunk++;
1715 countZero = nmaxmerge;
1716 previousChunk = outputChunk;
0f389141 1717 }
1718 }
1719 return merged;
1720}
1721
c57f56b7 1722//______________________________________________________________________________
1723Bool_t AliAnalysisAlien::MergeOutputs()
1724{
1725// Merge analysis outputs existing in the AliEn space.
1726 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1727 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1728 if (!Connect()) {
1729 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1730 return kFALSE;
0f389141 1731 }
c9e8f7fd 1732 if (fMergeViaJDL) {
1733 if (!TestBit(AliAnalysisGrid::kMerge)) {
1734 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
1735 return kFALSE;
1736 }
1737 if (fProductionMode) {
1738 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
1739 return kFALSE;
1740 }
0f389141 1741 Info("MergeOutputs", "Submitting merging JDL");
a03be957 1742 if (!SubmitMerging()) return kFALSE;
0f389141 1743 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 1744 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 1745 return kFALSE;
c57f56b7 1746 }
1747 // Get the output path
d2a409b2 1748 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1749 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1750 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1751 return kFALSE;
1752 }
1753 if (!fOutputFiles.Length()) {
1754 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1755 return kFALSE;
0f389141 1756 }
1757 // Check if fast read option was requested
e1c22e21 1758 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
1759 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
0f389141 1760 if (fFastReadOption) {
d3339be3 1761 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
1762 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
1763 gEnv->SetValue("XNet.ConnectTimeout",10);
1764 gEnv->SetValue("XNet.RequestTimeout",10);
0f389141 1765 gEnv->SetValue("XNet.MaxRedirectCount",2);
d3339be3 1766 gEnv->SetValue("XNet.ReconnectTimeout",10);
0f389141 1767 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 1768 }
e8b839ab 1769 // Make sure we change the temporary directory
1770 gSystem->Setenv("TMPDIR", gSystem->pwd());
149d288c 1771 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 1772 TIter next(list);
1773 TObjString *str;
a2f5fc01 1774 TString outputFile;
c57f56b7 1775 Bool_t merged = kTRUE;
1776 while((str=(TObjString*)next())) {
a2f5fc01 1777 outputFile = str->GetString();
1778 Int_t index = outputFile.Index("@");
1779 if (index > 0) outputFile.Remove(index);
1780 TString outputChunk = outputFile;
1781 outputChunk.ReplaceAll(".root", "_*.root");
319593fb 1782 // Skip already merged outputs
a2f5fc01 1783 if (!gSystem->AccessPathName(outputFile)) {
e1c22e21 1784 if (fOverwriteMode) {
a2f5fc01 1785 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
1786 gSystem->Unlink(outputFile);
1787 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 1788 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 1789 outputChunk.Data());
1790 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 1791 }
1792 } else {
a2f5fc01 1793 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
e1c22e21 1794 continue;
1795 }
1796 } else {
a2f5fc01 1797 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
e1c22e21 1798 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
a2f5fc01 1799 outputChunk.Data());
1800 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
e1c22e21 1801 }
1802 }
bb885a9e 1803 if (fMergeExcludes.Length() &&
a2f5fc01 1804 fMergeExcludes.Contains(outputFile.Data())) continue;
16a4353c 1805 // Perform a 'find' command in the output directory, looking for registered outputs
a2f5fc01 1806 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
0f389141 1807 if (!merged) {
1808 Error("MergeOutputs", "Terminate() will NOT be executed");
1809 return kFALSE;
ff07ec61 1810 }
1811 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
1812 if (fileOpened) fileOpened->Close();
c57f56b7 1813 }
0f389141 1814 return kTRUE;
c57f56b7 1815}
1816
bb885a9e 1817//______________________________________________________________________________
1818void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1819{
1820// Use the output files connected to output containers from the analysis manager
1821// rather than the files defined by SetOutputFiles
1822 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1823 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1824 analysis manager");
1825 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1826}
1827
149d288c 1828//______________________________________________________________________________
1829void AliAnalysisAlien::SetOutputFiles(const char *list)
1830{
1831// Manually set the output files list.
1832// Removes duplicates. Not allowed if default outputs are not disabled.
1833 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1834 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
1835 return;
1836 }
1837 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
1838 fOutputFiles = "";
1839 TString slist = list;
1840 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
1841 TObjArray *arr = slist.Tokenize(" ");
1842 TObjString *os;
1843 TIter next(arr);
1844 TString sout;
1845 while ((os=(TObjString*)next())) {
1846 sout = os->GetString();
1847 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1848 if (fOutputFiles.Contains(sout)) continue;
1849 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
1850 fOutputFiles += sout;
1851 }
1852 delete arr;
1853}
1854
1855//______________________________________________________________________________
1856void AliAnalysisAlien::SetOutputArchive(const char *list)
1857{
1858// Manually set the output archive list. Free text - you are on your own...
1859// Not allowed if default outputs are not disabled.
1860 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1861 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
1862 return;
1863 }
1864 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
1865 fOutputArchive = list;
1866}
1867
1868//______________________________________________________________________________
1869void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
1870{
1871// Setting a prefered output SE is not allowed anymore.
1872 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
1873}
1874
5513444a 1875//______________________________________________________________________________
1876Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1877{
1878// Start remote grid analysis.
1879
43da816a 1880 // Check if output files have to be taken from the analysis manager
1881 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1882 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1883 if (!mgr || !mgr->IsInitialized()) {
1884 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1885 return kFALSE;
43da816a 1886 }
1887 fOutputFiles = "";
1888 TIter next(mgr->GetOutputs());
1889 AliAnalysisDataContainer *output;
1890 while ((output=(AliAnalysisDataContainer*)next())) {
1891 const char *filename = output->GetFileName();
1892 if (!(strcmp(filename, "default"))) {
1893 if (!mgr->GetOutputEventHandler()) continue;
1894 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1895 }
84fcd93f 1896 if (fOutputFiles.Contains(filename)) continue;
149d288c 1897 if (fOutputFiles.Length()) fOutputFiles += ",";
43da816a 1898 fOutputFiles += filename;
1899 }
c07b9ce2 1900 // Add extra files registered to the analysis manager
1901 if (mgr->GetExtraFiles().Length()) {
149d288c 1902 if (fOutputFiles.Length()) fOutputFiles += ",";
1903 TString extra = mgr->GetExtraFiles();
1904 extra.ReplaceAll(" ", ",");
1905 // Protection in case extra files do not exist (will it work?)
1906 extra.ReplaceAll(".root", "*.root");
1907 fOutputFiles += extra;
84fcd93f 1908 }
149d288c 1909 // Compose the output archive.
1910 fOutputArchive = "log_archive.zip:std*@disk=1 ";
1911 fOutputArchive += Form("root_archive.zip:%s@disk=%d",fOutputFiles.Data(),fNreplicas);
43da816a 1912 }
f7b1cbc2 1913// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1914 if (TestBit(AliAnalysisGrid::kOffline)) {
1915 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1916 \n there nor any job run. You can revise the JDL and analysis \
1917 \n macro then run the same in \"submit\" mode.");
1918 } else if (TestBit(AliAnalysisGrid::kTest)) {
1919 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1920 \n dataset.");
1921 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1922 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1923 \n space and job submitted.");
1924 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1925 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 1926 if (fMergeViaJDL) CheckInputData();
5513444a 1927 return kTRUE;
c57f56b7 1928 } else {
1929 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1930 }
1931
348be253 1932 Print();
c57f56b7 1933 if (!Connect()) {
1934 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1935 return kFALSE;
16a4353c 1936 }
348be253 1937 if (IsCheckCopy()) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 1938 if (!CheckInputData()) {
1939 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1940 return kFALSE;
c57f56b7 1941 }
d3339be3 1942 if (!CreateDataset(fDataPattern)) {
1943 TString serror;
1944 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
1945 if (fRunNumbers.Length()) serror = "run numbers";
1946 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
1947 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
1948 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
1949 return kFALSE;
1950 }
c57f56b7 1951 WriteAnalysisFile();
1952 WriteAnalysisMacro();
1953 WriteExecutable();
1954 WriteValidationScript();
0f389141 1955 if (fMergeViaJDL) {
1956 WriteMergingMacro();
1957 WriteMergeExecutable();
1958 WriteValidationScript(kTRUE);
1959 }
5513444a 1960 if (!CreateJDL()) return kFALSE;
1961 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1962 if (TestBit(AliAnalysisGrid::kTest)) {
1963 // Locally testing the analysis
1964 Info("StartAnalysis", "\n_______________________________________________________________________ \
1965 \n Running analysis script in a daughter shell as on a worker node \
1966 \n_______________________________________________________________________");
149d288c 1967 TObjArray *list = fOutputFiles.Tokenize(",");
c57f56b7 1968 TIter next(list);
1969 TObjString *str;
a2f5fc01 1970 TString outputFile;
c57f56b7 1971 while((str=(TObjString*)next())) {
a2f5fc01 1972 outputFile = str->GetString();
1973 Int_t index = outputFile.Index("@");
1974 if (index > 0) outputFile.Remove(index);
1975 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
c57f56b7 1976 }
1977 delete list;
1978 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1979 TString validationScript = fExecutable;
1980 validationScript.ReplaceAll(".sh", "_validation.sh");
1981 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1982// gSystem->Exec("cat stdout");
5513444a 1983 return kFALSE;
c57f56b7 1984 }
5513444a 1985 // Check if submitting is managed by LPM manager
a3e84053 1986 if (fProductionMode) {
5513444a 1987 TString prodfile = fJDLName;
1988 prodfile.ReplaceAll(".jdl", ".prod");
1989 WriteProductionFile(prodfile);
1990 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1991 return kFALSE;
1992 }
a8739e8a 1993 // Submit AliEn job(s)
d2a409b2 1994 gGrid->Cd(fGridOutputDir);
a8739e8a 1995 TGridResult *res;
c57f56b7 1996 TString jobID = "";
d2a409b2 1997 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1998 // Submit a given xml or a set of runs
a8739e8a 1999 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 2000 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 2001 if (res) {
2002 const char *cjobId = res->GetKey(0,"jobId");
2003 if (!cjobId) {
a03be957 2004 gGrid->Stdout();
2005 gGrid->Stderr();
a8739e8a 2006 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 2007 return kFALSE;
a8739e8a 2008 } else {
2009 Info("StartAnalysis", "\n_______________________________________________________________________ \
2010 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2011 \n_______________________________________________________________________",
2012 fJDLName.Data(), cjobId);
2013 jobID = cjobId;
2014 }
2015 delete res;
a03be957 2016 } else {
2017 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2018 return kFALSE;
a8739e8a 2019 }
2020 } else {
d2a409b2 2021 // Submit for a range of enumeration of runs.
a03be957 2022 if (!Submit()) return kFALSE;
c57f56b7 2023 }
a8739e8a 2024
c57f56b7 2025 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2026 \n You may exit at any time and terminate the job later using the option <terminate> \
2027 \n ##################################################################################", jobID.Data());
bb885a9e 2028 gSystem->Exec("aliensh");
5513444a 2029 return kTRUE;
c57f56b7 2030}
2031
d2a409b2 2032//______________________________________________________________________________
a03be957 2033Bool_t AliAnalysisAlien::Submit()
d2a409b2 2034{
2035// Submit all master jobs.
2036 Int_t nmasterjobs = fInputFiles->GetEntries();
2037 Long_t tshoot = gSystem->Now();
a03be957 2038 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 2039 while (fNsubmitted < nmasterjobs) {
2040 Long_t now = gSystem->Now();
2041 if ((now-tshoot)>30000) {
2042 tshoot = now;
a03be957 2043 if (!SubmitNext()) return kFALSE;
d2a409b2 2044 }
2045 }
a03be957 2046 return kTRUE;
d2a409b2 2047}
2048
0f389141 2049//______________________________________________________________________________
a03be957 2050Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 2051{
2052// Submit all merging jobs.
2053 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2054 gGrid->Cd(fGridOutputDir);
2055 TString mergeJDLName = fExecutable;
2056 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2057 Int_t ntosubmit = fInputFiles->GetEntries();
2058 printf("### Submitting %d merging jobs...\n", ntosubmit);
2059 for (Int_t i=0; i<ntosubmit; i++) {
2060 TString query;
f866cba5 2061 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2062 runOutDir.ReplaceAll(".xml", "");
2063 if (fOutputToRunNo)
2064 query = Form("submit %s %s", mergeJDLName.Data(), runOutDir.Data());
2065 else
2066 query = Form("submit %s %03d", mergeJDLName.Data(), i);
0f389141 2067 printf("********* %s\n",query.Data());
2068 TGridResult *res = gGrid->Command(query);
2069 if (res) {
2070 const char *cjobId = res->GetKey(0,"jobId");
2071 if (!cjobId) {
a03be957 2072 gGrid->Stdout();
2073 gGrid->Stderr();
0f389141 2074 Error("StartAnalysis", "Your JDL %s could not be submitted", mergeJDLName.Data());
a03be957 2075 return kFALSE;
0f389141 2076 } else {
2077 Info("StartAnalysis", "\n_______________________________________________________________________ \
2078 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2079 \n_______________________________________________________________________",
2080 mergeJDLName.Data(), cjobId);
2081 }
2082 delete res;
a03be957 2083 } else {
2084 Error("SubmitMerging", "No grid result after submission !!! Bailing out...");
2085 return kFALSE;
0f389141 2086 }
2087 }
a03be957 2088 if (!ntosubmit) return kTRUE;
0f389141 2089 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
2090 \n You may exit at any time and terminate the job later using the option <terminate> but disabling SetMergeViaJDL\
2091 \n ##################################################################################");
2092 gSystem->Exec("aliensh");
a03be957 2093 return kTRUE;
0f389141 2094}
2095
d2a409b2 2096//______________________________________________________________________________
a03be957 2097Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2098{
2099// Submit next bunch of master jobs if the queue is free.
2100 static Bool_t iscalled = kFALSE;
2101 static Int_t firstmaster = 0;
2102 static Int_t lastmaster = 0;
2103 static Int_t npermaster = 0;
a03be957 2104 if (iscalled) return kTRUE;
d2a409b2 2105 iscalled = kTRUE;
2106 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2107 Int_t ntosubmit = 0;
2108 TGridResult *res;
2109 TString jobID = "";
2110 if (!fNsubmitted) ntosubmit = 1;
2111 else {
2112 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2113 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2114 // If last master not split, just return
a03be957 2115 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2116 // No more than 100 waiting jobs
a03be957 2117 if (nwaiting>100) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2118 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
2119 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
7586eedc 2120 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2121 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2122 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2123 }
2124 Int_t nmasterjobs = fInputFiles->GetEntries();
2125 for (Int_t i=0; i<ntosubmit; i++) {
2126 // Submit for a range of enumeration of runs.
a03be957 2127 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2128 TString query;
cd11251e 2129 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2130 runOutDir.ReplaceAll(".xml", "");
2131 if (fOutputToRunNo)
2132 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2133 else
2134 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2135 printf("********* %s\n",query.Data());
d2a409b2 2136 res = gGrid->Command(query);
2137 if (res) {
98ca124f 2138 TString cjobId1 = res->GetKey(0,"jobId");
2139 if (!cjobId1.Length()) {
d2a409b2 2140 iscalled = kFALSE;
a03be957 2141 gGrid->Stdout();
2142 gGrid->Stderr();
2143 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2144 return kFALSE;
d2a409b2 2145 } else {
2146 Info("StartAnalysis", "\n_______________________________________________________________________ \
2147 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2148 \n_______________________________________________________________________",
98ca124f 2149 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2150 jobID += cjobId1;
2151 jobID += " ";
98ca124f 2152 lastmaster = cjobId1.Atoi();
d2a409b2 2153 if (!firstmaster) firstmaster = lastmaster;
2154 fNsubmitted++;
2155 }
2156 delete res;
a03be957 2157 } else {
2158 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2159 return kFALSE;
d2a409b2 2160 }
2161 }
2162 iscalled = kFALSE;
a03be957 2163 return kTRUE;
d2a409b2 2164}
2165
c57f56b7 2166//______________________________________________________________________________
2167void AliAnalysisAlien::WriteAnalysisFile()
2168{
f10e8481 2169// Write current analysis manager into the file <analysisFile>
2170 TString analysisFile = fExecutable;
2171 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2172 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2173 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2174 if (!mgr || !mgr->IsInitialized()) {
2175 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2176 return;
2177 }
2178 // Check analysis type
2179 TObject *handler;
2180 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2181 handler = (TObject*)mgr->GetInputEventHandler();
2182 if (handler) {
2183 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2184 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2185 }
2186 TDirectory *cdir = gDirectory;
f10e8481 2187 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2188 if (file) {
e16a394c 2189 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2190 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
fe2d7fc2 2191 // Unless merging makes no sense
2192 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2193 mgr->Write();
2194 delete file;
fe2d7fc2 2195 // Enable termination for local jobs
2196 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2197 }
2198 if (cdir) cdir->cd();
f10e8481 2199 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2200 }
2201 Bool_t copy = kTRUE;
2202 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2203 if (copy) {
2204 CdWork();
2205 TString workdir = gGrid->GetHomeDirectory();
2206 workdir += fGridWorkingDir;
f10e8481 2207 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
2208 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
2209 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 2210 }
2211}
2212
2213//______________________________________________________________________________
2214void AliAnalysisAlien::WriteAnalysisMacro()
2215{
2216// Write the analysis macro that will steer the analysis in grid mode.
2217 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2218 ofstream out;
2219 out.open(fAnalysisMacro.Data(), ios::out);
2220 if (!out.good()) {
2221 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2222 return;
2223 }
5b273635 2224 Bool_t hasSTEERBase = kFALSE;
2225 Bool_t hasESD = kFALSE;
2226 Bool_t hasAOD = kFALSE;
2227 Bool_t hasANALYSIS = kFALSE;
2228 Bool_t hasANALYSISalice = kFALSE;
2229 Bool_t hasCORRFW = kFALSE;
c57f56b7 2230 TString func = fAnalysisMacro;
2231 TString type = "ESD";
2232 TString comment = "// Analysis using ";
2233 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
2234 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
2235 type = "AOD";
2236 comment += "AOD";
2237 }
0df6ccf2 2238 if (type!="AOD" && fFriendChainName!="") {
2239 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
2240 return;
2241 }
c57f56b7 2242 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
2243 else comment += " data";
2244 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
2245 func.ReplaceAll(".C", "");
2246 out << "void " << func.Data() << "()" << endl;
2247 out << "{" << endl;
2248 out << comment.Data() << endl;
2249 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 2250 out << " TStopwatch timer;" << endl;
2251 out << " timer.Start();" << endl << endl;
c57f56b7 2252 out << "// load base root libraries" << endl;
2253 out << " gSystem->Load(\"libTree\");" << endl;
2254 out << " gSystem->Load(\"libGeom\");" << endl;
2255 out << " gSystem->Load(\"libVMC\");" << endl;
2256 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 2257 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 2258 if (fAdditionalRootLibs.Length()) {
47a4137d 2259 // in principle libtree /lib geom libvmc etc. can go into this list, too
2260 out << "// Add aditional libraries" << endl;
2261 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2262 TIter next(list);
2263 TObjString *str;
2264 while((str=(TObjString*)next())) {
2265 if (str->GetString().Contains(".so"))
2266 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 2267 }
47a4137d 2268 if (list) delete list;
d5c6455a 2269 }
47a4137d 2270 out << "// include path" << endl;
2271 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2272 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
57377eb5 2273 out << "// Load analysis framework libraries" << endl;
4e5c5506 2274 if (!fPackages) {
4e5c5506 2275 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2276 out << " gSystem->Load(\"libESD\");" << endl;
2277 out << " gSystem->Load(\"libAOD\");" << endl;
2278 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 2279 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2280 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 2281 } else {
4e5c5506 2282 TIter next(fPackages);
2283 TObject *obj;
57377eb5 2284 TString pkgname;
5b273635 2285 TString setupPar = "AliAnalysisAlien::SetupPar";
57377eb5 2286 while ((obj=next())) {
2287 pkgname = obj->GetName();
4478e6f1 2288 if (pkgname == "STEERBase" ||
2289 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2290 if (pkgname == "ESD" ||
2291 pkgname == "ESD.par") hasESD = kTRUE;
2292 if (pkgname == "AOD" ||
2293 pkgname == "AOD.par") hasAOD = kTRUE;
2294 if (pkgname == "ANALYSIS" ||
2295 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2296 if (pkgname == "ANALYSISalice" ||
2297 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2298 if (pkgname == "CORRFW" ||
2299 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 2300 }
2301 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 2302 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2303 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 2304 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2305 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 2306 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2307 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 2308 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2309 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 2310 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2311 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 2312 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2313 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 2314 out << "// Compile other par packages" << endl;
2315 next.Reset();
fcc9bb6f 2316 while ((obj=next())) {
2317 pkgname = obj->GetName();
4478e6f1 2318 if (pkgname == "STEERBase" ||
2319 pkgname == "STEERBase.par" ||
2320 pkgname == "ESD" ||
2321 pkgname == "ESD.par" ||
2322 pkgname == "AOD" ||
2323 pkgname == "AOD.par" ||
2324 pkgname == "ANALYSIS" ||
2325 pkgname == "ANALYSIS.par" ||
2326 pkgname == "ANALYSISalice" ||
2327 pkgname == "ANALYSISalice.par" ||
2328 pkgname == "CORRFW" ||
2329 pkgname == "CORRFW.par") continue;
5b273635 2330 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 2331 }
4e5c5506 2332 }
6da75e0b 2333 if (fAdditionalLibs.Length()) {
2334 out << "// Add aditional AliRoot libraries" << endl;
2335 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2336 TIter next(list);
2337 TObjString *str;
2338 while((str=(TObjString*)next())) {
2339 if (str->GetString().Contains(".so"))
2340 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2341 }
2342 if (list) delete list;
2343 }
2344 out << endl;
c57f56b7 2345 out << "// analysis source to be compiled at runtime (if any)" << endl;
2346 if (fAnalysisSource.Length()) {
2347 TObjArray *list = fAnalysisSource.Tokenize(" ");
2348 TIter next(list);
2349 TObjString *str;
2350 while((str=(TObjString*)next())) {
2351 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2352 }
2353 if (list) delete list;
2354 }
2355 out << endl;
0f389141 2356 if (fFastReadOption) {
d3339be3 2357 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
2358 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 2359 out << "// fast xrootd reading enabled" << endl;
2360 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2361 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
2362 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 2363 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 2364 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 2365 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2366 }
e8b839ab 2367 // Change temp directory to current one
2368 out << "// Set temporary merging directory to current one" << endl;
2369 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
c57f56b7 2370 out << "// connect to AliEn and make the chain" << endl;
2371 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2372 if (IsUsingTags()) {
2373 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
2374 } else {
76535c98 2375 if(fFriendChainName!="AliAOD.VertexingHF.root") {
2376 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
2377 } else {
2378 out << " // Check if the macro to create the chain was provided" << endl;
2379 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
2380 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
2381 out << " return;" << endl;
2382 out << " }" << endl;
2383 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
2384 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
2385 }
c57f56b7 2386 }
2387 out << "// read the analysis manager from file" << endl;
f10e8481 2388 TString analysisFile = fExecutable;
2389 analysisFile.ReplaceAll(".sh", ".root");
2390 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 2391 out << " if (!file) return;" << endl;
2392 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2393 out << " AliAnalysisManager *mgr = 0;" << endl;
2394 out << " TKey *key;" << endl;
2395 out << " while ((key=(TKey*)nextkey())) {" << endl;
2396 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2397 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2398 out << " };" << endl;
2399 out << " if (!mgr) {" << endl;
f10e8481 2400 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 2401 out << " return;" << endl;
2402 out << " }" << endl << endl;
2403 out << " mgr->PrintStatus();" << endl;
52b6a92b 2404 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 2405 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 2406 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 2407 } else {
2408 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 2409 }
2410 }
c57f56b7 2411 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 2412 out << " timer.Stop();" << endl;
2413 out << " timer.Print();" << endl;
c57f56b7 2414 out << "}" << endl << endl;
2415 if (IsUsingTags()) {
2416 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
2417 out << "{" << endl;
2418 out << "// Create a chain using tags from the xml file." << endl;
2419 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
2420 out << " if (!coll) {" << endl;
2421 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2422 out << " return NULL;" << endl;
2423 out << " }" << endl;
2424 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
2425 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
2426 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
2427 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
2428 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
2429 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
2430 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
2431 out << " // Check if the cuts configuration file was provided" << endl;
2432 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
2433 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
2434 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2435 out << " }" << endl;
0df6ccf2 2436 if (fFriendChainName=="") {
2437 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2438 } else {
2439 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
2440 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
2441 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
2442 }
c57f56b7 2443 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
2444 out << " chain->ls();" << endl;
2445 out << " return chain;" << endl;
fcc9bb6f 2446 out << "}" << endl << endl;
c57f56b7 2447 if (gSystem->AccessPathName("ConfigureCuts.C")) {
2448 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
2449 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
2450 msg += " AliLHCTagCuts *lhcCuts,\n";
2451 msg += " AliDetectorTagCuts *detCuts,\n";
2452 msg += " AliEventTagCuts *evCuts)";
2453 Info("WriteAnalysisMacro", msg.Data());
2454 }
0df6ccf2 2455 }
2456 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 2457 out <<"//________________________________________________________________________________" << endl;
c57f56b7 2458 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
2459 out << "{" << endl;
2460 out << "// Create a chain using url's from xml file" << endl;
2461 out << " TString treename = type;" << endl;
2462 out << " treename.ToLower();" << endl;
2463 out << " treename += \"Tree\";" << endl;
e02fee64 2464 out << " printf(\"***************************************\\n\");" << endl;
2465 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
2466 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 2467 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
2468 out << " if (!coll) {" << endl;
2469 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2470 out << " return NULL;" << endl;
2471 out << " }" << endl;
2472 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 2473 if(fFriendChainName!="") {
2474 out << " TChain *chainFriend = new TChain(treename);" << endl;
2475 }
c57f56b7 2476 out << " coll->Reset();" << endl;
0df6ccf2 2477 out << " while (coll->Next()) {" << endl;
2478 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
2479 if(fFriendChainName!="") {
2480 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
2481 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2482 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2483 out << " chainFriend->Add(fileFriend.Data());" << endl;
2484 }
2485 out << " }" << endl;
c57f56b7 2486 out << " if (!chain->GetNtrees()) {" << endl;
2487 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
2488 out << " return NULL;" << endl;
2489 out << " }" << endl;
0df6ccf2 2490 if(fFriendChainName!="") {
2491 out << " chain->AddFriend(chainFriend);" << endl;
2492 }
c57f56b7 2493 out << " return chain;" << endl;
fcc9bb6f 2494 out << "}" << endl << endl;
c57f56b7 2495 }
5b273635 2496 if (hasANALYSISalice) {
2497 out <<"//________________________________________________________________________________" << endl;
2498 out << "Bool_t SetupPar(const char *package) {" << endl;
2499 out << "// Compile the package and set it up." << endl;
2500 out << " TString pkgdir = package;" << endl;
2501 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2502 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2503 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2504 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2505 out << " // Check for BUILD.sh and execute" << endl;
2506 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2507 out << " printf(\"*******************************\\n\");" << endl;
2508 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2509 out << " printf(\"*******************************\\n\");" << endl;
2510 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2511 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2512 out << " gSystem->ChangeDirectory(cdir);" << endl;
2513 out << " return kFALSE;" << endl;
2514 out << " }" << endl;
2515 out << " } else {" << endl;
2516 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2517 out << " gSystem->ChangeDirectory(cdir);" << endl;
2518 out << " return kFALSE;" << endl;
2519 out << " }" << endl;
2520 out << " // Check for SETUP.C and execute" << endl;
2521 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2522 out << " printf(\"*******************************\\n\");" << endl;
2523 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2524 out << " printf(\"*******************************\\n\");" << endl;
2525 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2526 out << " } else {" << endl;
2527 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2528 out << " gSystem->ChangeDirectory(cdir);" << endl;
2529 out << " return kFALSE;" << endl;
2530 out << " }" << endl;
2531 out << " // Restore original workdir" << endl;
2532 out << " gSystem->ChangeDirectory(cdir);" << endl;
2533 out << " return kTRUE;" << endl;
2534 out << "}" << endl;
2535 }
c57f56b7 2536 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2537 }
2538 Bool_t copy = kTRUE;
2539 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2540 if (copy) {
2541 CdWork();
2542 TString workdir = gGrid->GetHomeDirectory();
2543 workdir += fGridWorkingDir;
2544 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2545 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2546 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2547 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2548 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2549 }
2550 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2551 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2552 }
2553}
2554
0f389141 2555//______________________________________________________________________________
2556void AliAnalysisAlien::WriteMergingMacro()
2557{
2558// Write a macro to merge the outputs per master job.
2559 if (!fMergeViaJDL) return;
2560 if (!fOutputFiles.Length()) {
2561 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2562 return;
2563 }
2564 TString mergingMacro = fExecutable;
2565 mergingMacro.ReplaceAll(".sh","_merge.C");
2566 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2567 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2568 ofstream out;
2569 out.open(mergingMacro.Data(), ios::out);
2570 if (!out.good()) {
2571 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2572 return;
2573 }
5b273635 2574 Bool_t hasSTEERBase = kFALSE;
2575 Bool_t hasESD = kFALSE;
2576 Bool_t hasAOD = kFALSE;
2577 Bool_t hasANALYSIS = kFALSE;
2578 Bool_t hasANALYSISalice = kFALSE;
2579 Bool_t hasCORRFW = kFALSE;
0f389141 2580 TString func = mergingMacro;
2581 TString comment;
2582 func.ReplaceAll(".C", "");
2583 out << "void " << func.Data() << "(const char *dir)" << endl;
2584 out << "{" << endl;
2585 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
2586 out << " TStopwatch timer;" << endl;
2587 out << " timer.Start();" << endl << endl;
2588 out << "// load base root libraries" << endl;
2589 out << " gSystem->Load(\"libTree\");" << endl;
2590 out << " gSystem->Load(\"libGeom\");" << endl;
2591 out << " gSystem->Load(\"libVMC\");" << endl;
2592 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
2593 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
2594 if (fAdditionalRootLibs.Length()) {
2595 // in principle libtree /lib geom libvmc etc. can go into this list, too
2596 out << "// Add aditional libraries" << endl;
2597 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2598 TIter next(list);
2599 TObjString *str;
2600 while((str=(TObjString*)next())) {
2601 if (str->GetString().Contains(".so"))
2602 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2603 }
2604 if (list) delete list;
2605 }
2606 out << "// include path" << endl;
2607 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2608 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
2609 out << "// Load analysis framework libraries" << endl;
2610 if (!fPackages) {
2611 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2612 out << " gSystem->Load(\"libESD\");" << endl;
2613 out << " gSystem->Load(\"libAOD\");" << endl;
2614 out << " gSystem->Load(\"libANALYSIS\");" << endl;
2615 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2616 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
2617 } else {
2618 TIter next(fPackages);
2619 TObject *obj;
2620 TString pkgname;
5b273635 2621 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 2622 while ((obj=next())) {
2623 pkgname = obj->GetName();
2624 if (pkgname == "STEERBase" ||
2625 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2626 if (pkgname == "ESD" ||
2627 pkgname == "ESD.par") hasESD = kTRUE;
2628 if (pkgname == "AOD" ||
2629 pkgname == "AOD.par") hasAOD = kTRUE;
2630 if (pkgname == "ANALYSIS" ||
2631 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2632 if (pkgname == "ANALYSISalice" ||
2633 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2634 if (pkgname == "CORRFW" ||
2635 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
2636 }
5b273635 2637 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 2638 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2639 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 2640 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2641 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 2642 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2643 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 2644 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2645 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 2646 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2647 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 2648 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2649 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 2650 out << "// Compile other par packages" << endl;
2651 next.Reset();
2652 while ((obj=next())) {
2653 pkgname = obj->GetName();
2654 if (pkgname == "STEERBase" ||
2655 pkgname == "STEERBase.par" ||
2656 pkgname == "ESD" ||
2657 pkgname == "ESD.par" ||
2658 pkgname == "AOD" ||
2659 pkgname == "AOD.par" ||
2660 pkgname == "ANALYSIS" ||
2661 pkgname == "ANALYSIS.par" ||
2662 pkgname == "ANALYSISalice" ||
2663 pkgname == "ANALYSISalice.par" ||
2664 pkgname == "CORRFW" ||
2665 pkgname == "CORRFW.par") continue;
5b273635 2666 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 2667 }
2668 }
2669 if (fAdditionalLibs.Length()) {
2670 out << "// Add aditional AliRoot libraries" << endl;
2671 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2672 TIter next(list);
2673 TObjString *str;
2674 while((str=(TObjString*)next())) {
2675 if (str->GetString().Contains(".so"))
2676 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2677 }
2678 if (list) delete list;
2679 }
2680 out << endl;
2681 out << "// Analysis source to be compiled at runtime (if any)" << endl;
2682 if (fAnalysisSource.Length()) {
2683 TObjArray *list = fAnalysisSource.Tokenize(" ");
2684 TIter next(list);
2685 TObjString *str;
2686 while((str=(TObjString*)next())) {
2687 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2688 }
2689 if (list) delete list;
2690 }
149d288c 2691 out << endl;
2692
0f389141 2693 if (fFastReadOption) {
2694 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
2695 out << "// fast xrootd reading enabled" << endl;
2696 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2697 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
2698 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 2699 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 2700 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 2701 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
e8b839ab 2702 }
2703 // Change temp directory to current one
2704 out << "// Set temporary merging directory to current one" << endl;
2705 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
0f389141 2706 out << "// Connect to AliEn" << endl;
2707 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2708 out << " TString outputDir = \"" << fGridOutputDir << "/\";" << endl;
2709 out << " outputDir += dir;" << endl;
2710 out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
2711 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
1f0d1ca2 2712 out << " mergeExcludes += \"" << AliAnalysisManager::GetAnalysisManager()->GetExtraFiles() << "\";" << endl;
0f389141 2713 out << " TObjArray *list = outputFiles.Tokenize(\" \");" << endl;
2714 out << " TIter *iter = new TIter(list);" << endl;
2715 out << " TObjString *str;" << endl;
a2f5fc01 2716 out << " TString outputFile;" << endl;
0f389141 2717 out << " Bool_t merged = kTRUE;" << endl;
2718 out << " while((str=(TObjString*)iter->Next())) {" << endl;
a2f5fc01 2719 out << " outputFile = str->GetString();" << endl;
2720 out << " Int_t index = outputFile.Index(\"@\");" << endl;
2721 out << " if (index > 0) outputFile.Remove(index);" << endl;
0f389141 2722 out << " // Skip already merged outputs" << endl;
a2f5fc01 2723 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
2724 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
0f389141 2725 out << " continue;" << endl;
2726 out << " }" << endl;
a2f5fc01 2727 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
2728 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ");" << endl;
0f389141 2729 out << " if (!merged) {" << endl;
a2f5fc01 2730 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
0f389141 2731 out << " }" << endl;
2732 out << " }" << endl;
f866cba5 2733 out << "// read the analysis manager from file" << endl;
2734 TString analysisFile = fExecutable;
2735 analysisFile.ReplaceAll(".sh", ".root");
2736 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
2737 out << " if (!file) return;" << endl;
2738 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2739 out << " AliAnalysisManager *mgr = 0;" << endl;
2740 out << " TKey *key;" << endl;
2741 out << " while ((key=(TKey*)nextkey())) {" << endl;
2742 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2743 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2744 out << " };" << endl;
2745 out << " if (!mgr) {" << endl;
2746 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
2747 out << " return;" << endl;
2748 out << " }" << endl << endl;
b385fec0 2749 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 2750 out << " mgr->PrintStatus();" << endl;
2751 if (AliAnalysisManager::GetAnalysisManager()) {
2752 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
2753 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
2754 } else {
2755 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
2756 }
2757 }
2758 out << " mgr->StartAnalysis(\"gridterminate\");" << endl;
0f389141 2759 out << "}" << endl << endl;
5b273635 2760 if (hasANALYSISalice) {
2761 out <<"//________________________________________________________________________________" << endl;
2762 out << "Bool_t SetupPar(const char *package) {" << endl;
2763 out << "// Compile the package and set it up." << endl;
2764 out << " TString pkgdir = package;" << endl;
2765 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2766 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2767 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2768 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2769 out << " // Check for BUILD.sh and execute" << endl;
2770 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2771 out << " printf(\"*******************************\\n\");" << endl;
2772 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2773 out << " printf(\"*******************************\\n\");" << endl;
2774 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2775 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2776 out << " gSystem->ChangeDirectory(cdir);" << endl;
2777 out << " return kFALSE;" << endl;
2778 out << " }" << endl;
2779 out << " } else {" << endl;
2780 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2781 out << " gSystem->ChangeDirectory(cdir);" << endl;
2782 out << " return kFALSE;" << endl;
2783 out << " }" << endl;
2784 out << " // Check for SETUP.C and execute" << endl;
2785 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2786 out << " printf(\"*******************************\\n\");" << endl;
2787 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2788 out << " printf(\"*******************************\\n\");" << endl;
2789 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2790 out << " } else {" << endl;
2791 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2792 out << " gSystem->ChangeDirectory(cdir);" << endl;
2793 out << " return kFALSE;" << endl;
2794 out << " }" << endl;
2795 out << " // Restore original workdir" << endl;
2796 out << " gSystem->ChangeDirectory(cdir);" << endl;
2797 out << " return kTRUE;" << endl;
2798 out << "}" << endl;
2799 }
0f389141 2800 }
2801 Bool_t copy = kTRUE;
2802 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2803 if (copy) {
2804 CdWork();
2805 TString workdir = gGrid->GetHomeDirectory();
2806 workdir += fGridWorkingDir;
2807 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
2808 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
2809 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
2810 }
2811}
2812
2813//______________________________________________________________________________
2814Bool_t AliAnalysisAlien::SetupPar(const char *package)
2815{
2816// Compile the par file archive pointed by <package>. This must be present in the current durectory.
2817// Note that for loading the compiled library. The current directory should have precedence in
2818// LD_LIBRARY_PATH
2819 TString pkgdir = package;
2820 pkgdir.ReplaceAll(".par","");
2821 gSystem->Exec(Form("tar xvzf %s.par", pkgdir.Data()));
2822 TString cdir = gSystem->WorkingDirectory();
2823 gSystem->ChangeDirectory(pkgdir);
2824 // Check for BUILD.sh and execute
2825 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
2826 printf("**************************************************\n");
2827 printf("*** Building PAR archive %s\n", package);
2828 printf("**************************************************\n");
2829 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
2830 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
2831 gSystem->ChangeDirectory(cdir);
2832 return kFALSE;
2833 }
2834 } else {
2835 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
2836 gSystem->ChangeDirectory(cdir);
2837 return kFALSE;
2838 }
2839 // Check for SETUP.C and execute
2840 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
2841 printf("**************************************************\n");
2842 printf("*** Setup PAR archive %s\n", package);
2843 printf("**************************************************\n");
2844 gROOT->Macro("PROOF-INF/SETUP.C");
2845 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
2846 } else {
2847 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
2848 gSystem->ChangeDirectory(cdir);
2849 return kFALSE;
2850 }
2851 // Restore original workdir
2852 gSystem->ChangeDirectory(cdir);
2853 return kTRUE;
2854}
2855
c57f56b7 2856//______________________________________________________________________________
2857void AliAnalysisAlien::WriteExecutable()
2858{
2859// Generate the alien executable script.
2860 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2861 ofstream out;
2862 out.open(fExecutable.Data(), ios::out);
2863 if (out.bad()) {
5513444a 2864 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2865 return;
2866 }
2867 out << "#!/bin/bash" << endl;
c57f56b7 2868 out << "echo \"=========================================\"" << endl;
2869 out << "echo \"############## PATH : ##############\"" << endl;
2870 out << "echo $PATH" << endl;
2871 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2872 out << "echo $LD_LIBRARY_PATH" << endl;
2873 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2874 out << "echo $ROOTSYS" << endl;
2875 out << "echo \"############## which root : ##############\"" << endl;
2876 out << "which root" << endl;
2877 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2878 out << "echo $ALICE_ROOT" << endl;
2879 out << "echo \"############## which aliroot : ##############\"" << endl;
2880 out << "which aliroot" << endl;
9c5ddadc 2881 out << "echo \"############## system limits : ##############\"" << endl;
2882 out << "ulimit -a" << endl;
2883 out << "echo \"############## memory : ##############\"" << endl;
2884 out << "free -m" << endl;
c57f56b7 2885 out << "echo \"=========================================\"" << endl << endl;
74a53467 2886 // Make sure we can properly compile par files
2887 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2888 out << fExecutableCommand << " ";
631c0b05 2889 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2890 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2891 out << "echo \"############## memory after: ##############\"" << endl;
2892 out << "free -m" << endl;
c57f56b7 2893 }
2894 Bool_t copy = kTRUE;
2895 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2896 if (copy) {
2897 CdWork();
2898 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2899 TString bindir = Form("%s/bin", workdir.Data());
2900 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2901 workdir += fGridWorkingDir;
2902 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2903 if (FileExists(executable)) gGrid->Rm(executable);
2904 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2905 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2906 }
2907}
2908
0f389141 2909//______________________________________________________________________________
2910void AliAnalysisAlien::WriteMergeExecutable()
2911{
2912// Generate the alien executable script for the merging job.
2913 if (!fMergeViaJDL) return;
2914 TString mergeExec = fExecutable;
2915 mergeExec.ReplaceAll(".sh", "_merge.sh");
2916 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2917 ofstream out;
2918 out.open(mergeExec.Data(), ios::out);
2919 if (out.bad()) {
2920 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
2921 return;
2922 }
2923 out << "#!/bin/bash" << endl;
2924 out << "echo \"=========================================\"" << endl;
2925 out << "echo \"############## PATH : ##############\"" << endl;
2926 out << "echo $PATH" << endl;
2927 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2928 out << "echo $LD_LIBRARY_PATH" << endl;
2929 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2930 out << "echo $ROOTSYS" << endl;
2931 out << "echo \"############## which root : ##############\"" << endl;
2932 out << "which root" << endl;
2933 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2934 out << "echo $ALICE_ROOT" << endl;
2935 out << "echo \"############## which aliroot : ##############\"" << endl;
2936 out << "which aliroot" << endl;
2937 out << "echo \"############## system limits : ##############\"" << endl;
2938 out << "ulimit -a" << endl;
2939 out << "echo \"############## memory : ##############\"" << endl;
2940 out << "free -m" << endl;
2941 out << "echo \"=========================================\"" << endl << endl;
2942 // Make sure we can properly compile par files
2943 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
2944 TString mergeMacro = fExecutable;
2945 mergeMacro.ReplaceAll(".sh", "_merge.C");
2946 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
2947 out << fExecutableCommand << " " << "$ARG" << endl;
2948 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
2949 out << "echo \"############## memory after: ##############\"" << endl;
2950 out << "free -m" << endl;
0f389141 2951 }
2952 Bool_t copy = kTRUE;
2953 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2954 if (copy) {
2955 CdWork();
2956 TString workdir = gGrid->GetHomeDirectory();
2957 TString bindir = Form("%s/bin", workdir.Data());
2958 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
2959 workdir += fGridWorkingDir;
2960 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
2961 if (FileExists(executable)) gGrid->Rm(executable);
2962 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
2963 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
2964 }
2965}
2966
c57f56b7 2967//______________________________________________________________________________
5513444a 2968void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2969{
2970// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2971// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2972// Next lines: full_path_to_dataset XXX (XXX is a string)
2973// To submit, one has to: submit jdl XXX for all lines
2974 ofstream out;
2975 out.open(filename, ios::out);
2976 if (out.bad()) {
2977 Error("WriteProductionFile", "Bad file name: %s", filename);
2978 return;
2979 }
2980 TString workdir = gGrid->GetHomeDirectory();
2981 workdir += fGridWorkingDir;
f5e8c702 2982 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2983 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2984 out << locjdl << " " << njobspermaster << endl;
5513444a 2985 Int_t nmasterjobs = fInputFiles->GetEntries();
2986 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 2987 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2988 runOutDir.ReplaceAll(".xml", "");
2989 if (fOutputToRunNo)
2990 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
2991 else
2992 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2993 }
2994 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
fdbbc7be 2995 if (FileExists(filename)) gGrid->Rm(filename);
5513444a 2996 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2997}
2998
2999//______________________________________________________________________________
0f389141 3000void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 3001{
3002// Generate the alien validation script.
3003 // Generate the validation script
3004 TObjString *os;
0d5d317c 3005 TString validationScript = fExecutable;
0f389141 3006 if (merge) validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
3007 else validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 3008 if (!Connect()) {
3009 Error("WriteValidationScript", "Alien connection required");
3010 return;
3011 }
a2f5fc01 3012 TString outStream = "";
3013 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
c57f56b7 3014 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3015 ofstream out;
0d5d317c 3016 out.open(validationScript, ios::out);
c57f56b7 3017 out << "#!/bin/bash" << endl;
3018 out << "##################################################" << endl;
3019 out << "validateout=`dirname $0`" << endl;
3020 out << "validatetime=`date`" << endl;
3021 out << "validated=\"0\";" << endl;
3022 out << "error=0" << endl;
3023 out << "if [ -z $validateout ]" << endl;
3024 out << "then" << endl;
3025 out << " validateout=\".\"" << endl;
3026 out << "fi" << endl << endl;
3027 out << "cd $validateout;" << endl;
3028 out << "validateworkdir=`pwd`;" << endl << endl;
a2f5fc01 3029 out << "echo \"*******************************************************\"" << outStream << endl;
3030 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
c57f56b7 3031 out << "" << endl;
a2f5fc01 3032 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3033 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3034 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3035 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3036 out << "ls -la ./" << outStream << endl;
3037 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
c57f56b7 3038 out << "##################################################" << endl;
ebec370a 3039 out << "" << endl;
3040
3041 out << "if [ ! -f stderr ] ; then" << endl;
3042 out << " error=1" << endl;
a2f5fc01 3043 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3044 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3045 out << "fi" << endl;
3046
b34c9f51 3047 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3048 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3049 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3050 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3051 out << "" << endl;
3052
ebec370a 3053 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3054 out << " error=1" << endl;
a2f5fc01 3055 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3056 out << " echo \"$parArch\" " << outStream << endl;
3057 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3058 out << "fi" << endl;
3059
3060 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3061 out << " error=1" << endl;
a2f5fc01 3062 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3063 out << " echo \"$segViol\" " << outStream << endl;
3064 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3065 out << "fi" << endl;
3066
3067 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3068 out << " error=1" << endl;
a2f5fc01 3069 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3070 out << " echo \"$segFault\" " << outStream << endl;
3071 out << " echo \"Error = $error\" " << outStream << endl;
ebec370a 3072 out << "fi" << endl;
3073
b34c9f51 3074 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3075 out << " error=1" << endl;
a2f5fc01 3076 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3077 out << " echo \"$glibcErr\" " << outStream << endl;
3078 out << " echo \"Error = $error\" " << outStream << endl;
b34c9f51 3079 out << "fi" << endl;
3080
ebec370a 3081 // Part dedicated to the specific analyses running into the train
3082
149d288c 3083 TObjArray *arr = fOutputFiles.Tokenize(",");
c57f56b7 3084 TIter next1(arr);
a2f5fc01 3085 TString outputFile;
5b9b4998 3086 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3087 TString extra = mgr->GetExtraFiles();
c57f56b7 3088 while ((os=(TObjString*)next1())) {
a2f5fc01 3089 outputFile = os->GetString();
3090 Int_t index = outputFile.Index("@");
3091 if (index > 0) outputFile.Remove(index);
3092 if (merge && fMergeExcludes.Contains(outputFile)) continue;
3093 if (extra.Contains(outputFile)) continue;
3094 if (outputFile.Contains("*")) continue;
3095 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
c57f56b7 3096 out << " error=1" << endl;
a2f5fc01 3097 out << " echo \"Output file(s) not found. Job FAILED !\"" << outStream << endl;
c57f56b7 3098 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
3099 out << "fi" << endl;
3100 }
3101 delete arr;
f866cba5 3102 if (!merge) {
3103 out << "if ! [ -f outputs_valid ] ; then" << endl;
3104 out << " error=1" << endl;
3105 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3106 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3107 out << "fi" << endl;
3108 }
923e2ca5 3109
c57f56b7 3110 out << "if [ $error = 0 ] ; then" << endl;
a2f5fc01 3111 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
149d288c 3112 if (!IsKeepLogs()) {
3113 out << " echo \"* === Logs std* will be deleted === \"" << endl;
a2f5fc01 3114 outStream = "";
149d288c 3115 out << " rm -f std*" << endl;
3116 }
c57f56b7 3117 out << "fi" << endl;
3118
a2f5fc01 3119 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3120 out << "echo \"*******************************************************\"" << outStream << endl;
c57f56b7 3121 out << "cd -" << endl;
3122 out << "exit $error" << endl;
3123 }
3124 Bool_t copy = kTRUE;
3125 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3126 if (copy) {
3127 CdWork();
3128 TString workdir = gGrid->GetHomeDirectory();
3129 workdir += fGridWorkingDir;
0d5d317c 3130 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
3131 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3132 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3133 }
3134}