]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Modified computation of innermost ITS layer to be considered in AliITStrackerMI:...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
25#include "TError.h"
c57f56b7 26#include "TROOT.h"
27#include "TSystem.h"
28#include "TFile.h"
29#include "TObjString.h"
30#include "TObjArray.h"
31#include "TGrid.h"
32#include "TGridResult.h"
33#include "TGridCollection.h"
34#include "TGridJDL.h"
d2a409b2 35#include "TGridJobStatusList.h"
36#include "TGridJobStatus.h"
c57f56b7 37#include "TFileMerger.h"
38#include "AliAnalysisManager.h"
bb885a9e 39#include "AliVEventHandler.h"
40#include "AliAnalysisDataContainer.h"
c57f56b7 41#include "AliAnalysisAlien.h"
42
43ClassImp(AliAnalysisAlien)
44
45//______________________________________________________________________________
46AliAnalysisAlien::AliAnalysisAlien()
47 :AliAnalysisGrid(),
48 fGridJDL(NULL),
0f389141 49 fMergingJDL(NULL),
c57f56b7 50 fPrice(0),
51 fTTL(0),
52 fSplitMaxInputFileNumber(0),
53 fMaxInitFailed(0),
54 fMasterResubmitThreshold(0),
bb885a9e 55 fNtestFiles(0),
319593fb 56 fNrunsPerMaster(0),
16a4353c 57 fMaxMergeFiles(0),
d2a409b2 58 fNsubmitted(0),
a3e84053 59 fProductionMode(0),
cd11251e 60 fOutputToRunNo(0),
0f389141 61 fMergeViaJDL(0),
62 fFastReadOption(0),
63 fOverwriteMode(0),
c57f56b7 64 fRunNumbers(),
65 fExecutable(),
0a1c1f7f 66 fExecutableCommand(),
c57f56b7 67 fArguments(),
631c0b05 68 fExecutableArgs(),
c57f56b7 69 fAnalysisMacro(),
70 fAnalysisSource(),
d5c6455a 71 fAdditionalRootLibs(),
c57f56b7 72 fAdditionalLibs(),
73 fSplitMode(),
74 fAPIVersion(),
75 fROOTVersion(),
76 fAliROOTVersion(),
648174cf 77 fExternalPackages(),
c57f56b7 78 fUser(),
79 fGridWorkingDir(),
80 fGridDataDir(),
81 fDataPattern(),
82 fGridOutputDir(),
83 fOutputArchive(),
84 fOutputFiles(),
85 fInputFormat(),
e7c71df0 86 fDatasetName(),
c57f56b7 87 fJDLName(),
bb885a9e 88 fMergeExcludes(),
f965131e 89 fIncludePath(),
bb885a9e 90 fCloseSE(),
0df6ccf2 91 fFriendChainName(),
c6cb3634 92 fJobTag(),
648174cf 93 fOutputSingle(),
5fce53f4 94 fRunPrefix(),
4e5c5506 95 fInputFiles(0),
96 fPackages(0)
c57f56b7 97{
98// Dummy ctor.
99 SetDefaults();
100}
101
102//______________________________________________________________________________
103AliAnalysisAlien::AliAnalysisAlien(const char *name)
104 :AliAnalysisGrid(name),
105 fGridJDL(NULL),
0f389141 106 fMergingJDL(NULL),
c57f56b7 107 fPrice(0),
108 fTTL(0),
109 fSplitMaxInputFileNumber(0),
110 fMaxInitFailed(0),
111 fMasterResubmitThreshold(0),
bb885a9e 112 fNtestFiles(0),
319593fb 113 fNrunsPerMaster(0),
16a4353c 114 fMaxMergeFiles(0),
d2a409b2 115 fNsubmitted(0),
a3e84053 116 fProductionMode(0),
cd11251e 117 fOutputToRunNo(0),
0f389141 118 fMergeViaJDL(0),
119 fFastReadOption(0),
120 fOverwriteMode(0),
c57f56b7 121 fRunNumbers(),
122 fExecutable(),
0a1c1f7f 123 fExecutableCommand(),
c57f56b7 124 fArguments(),
631c0b05 125 fExecutableArgs(),
c57f56b7 126 fAnalysisMacro(),
127 fAnalysisSource(),
d5c6455a 128 fAdditionalRootLibs(),
c57f56b7 129 fAdditionalLibs(),
130 fSplitMode(),
131 fAPIVersion(),
132 fROOTVersion(),
133 fAliROOTVersion(),
648174cf 134 fExternalPackages(),
c57f56b7 135 fUser(),
136 fGridWorkingDir(),
137 fGridDataDir(),
138 fDataPattern(),
139 fGridOutputDir(),
140 fOutputArchive(),
141 fOutputFiles(),
142 fInputFormat(),
e7c71df0 143 fDatasetName(),
c57f56b7 144 fJDLName(),
bb885a9e 145 fMergeExcludes(),
f965131e 146 fIncludePath(),
bb885a9e 147 fCloseSE(),
0df6ccf2 148 fFriendChainName(),
c6cb3634 149 fJobTag(),
648174cf 150 fOutputSingle(),
5fce53f4 151 fRunPrefix(),
4e5c5506 152 fInputFiles(0),
153 fPackages(0)
c57f56b7 154{
155// Default ctor.
156 SetDefaults();
157}
158
159//______________________________________________________________________________
160AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
161 :AliAnalysisGrid(other),
162 fGridJDL(NULL),
0f389141 163 fMergingJDL(NULL),
c57f56b7 164 fPrice(other.fPrice),
165 fTTL(other.fTTL),
166 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
167 fMaxInitFailed(other.fMaxInitFailed),
168 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 169 fNtestFiles(other.fNtestFiles),
319593fb 170 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 171 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 172 fNsubmitted(other.fNsubmitted),
a3e84053 173 fProductionMode(other.fProductionMode),
cd11251e 174 fOutputToRunNo(other.fOutputToRunNo),
0f389141 175 fMergeViaJDL(other.fMergeViaJDL),
176 fFastReadOption(other.fFastReadOption),
177 fOverwriteMode(other.fOverwriteMode),
c57f56b7 178 fRunNumbers(other.fRunNumbers),
179 fExecutable(other.fExecutable),
0a1c1f7f 180 fExecutableCommand(other.fExecutableCommand),
c57f56b7 181 fArguments(other.fArguments),
631c0b05 182 fExecutableArgs(other.fExecutableArgs),
c57f56b7 183 fAnalysisMacro(other.fAnalysisMacro),
184 fAnalysisSource(other.fAnalysisSource),
d5c6455a 185 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 186 fAdditionalLibs(other.fAdditionalLibs),
187 fSplitMode(other.fSplitMode),
188 fAPIVersion(other.fAPIVersion),
189 fROOTVersion(other.fROOTVersion),
190 fAliROOTVersion(other.fAliROOTVersion),
648174cf 191 fExternalPackages(other.fExternalPackages),
c57f56b7 192 fUser(other.fUser),
193 fGridWorkingDir(other.fGridWorkingDir),
194 fGridDataDir(other.fGridDataDir),
195 fDataPattern(other.fDataPattern),
196 fGridOutputDir(other.fGridOutputDir),
197 fOutputArchive(other.fOutputArchive),
198 fOutputFiles(other.fOutputFiles),
199 fInputFormat(other.fInputFormat),
e7c71df0 200 fDatasetName(other.fDatasetName),
c57f56b7 201 fJDLName(other.fJDLName),
bb885a9e 202 fMergeExcludes(other.fMergeExcludes),
f965131e 203 fIncludePath(other.fIncludePath),
bb885a9e 204 fCloseSE(other.fCloseSE),
0df6ccf2 205 fFriendChainName(other.fFriendChainName),
c6cb3634 206 fJobTag(other.fJobTag),
648174cf 207 fOutputSingle(other.fOutputSingle),
5fce53f4 208 fRunPrefix(other.fRunPrefix),
4e5c5506 209 fInputFiles(0),
210 fPackages(0)
c57f56b7 211{
212// Copy ctor.
213 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 214 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 215 fRunRange[0] = other.fRunRange[0];
216 fRunRange[1] = other.fRunRange[1];
c57f56b7 217 if (other.fInputFiles) {
218 fInputFiles = new TObjArray();
219 TIter next(other.fInputFiles);
220 TObject *obj;
221 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
222 fInputFiles->SetOwner();
223 }
4e5c5506 224 if (other.fPackages) {
225 fPackages = new TObjArray();
226 TIter next(other.fPackages);
227 TObject *obj;
228 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
229 fPackages->SetOwner();
230 }
c57f56b7 231}
232
233//______________________________________________________________________________
234AliAnalysisAlien::~AliAnalysisAlien()
235{
236// Destructor.
237 if (fGridJDL) delete fGridJDL;
0f389141 238 if (fMergingJDL) delete fMergingJDL;
c57f56b7 239 if (fInputFiles) delete fInputFiles;
4e5c5506 240 if (fPackages) delete fPackages;
c57f56b7 241}
242
243//______________________________________________________________________________
244AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
245{
246// Assignment.
247 if (this != &other) {
248 AliAnalysisGrid::operator=(other);
249 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 250 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 251 fPrice = other.fPrice;
252 fTTL = other.fTTL;
253 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
254 fMaxInitFailed = other.fMaxInitFailed;
255 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 256 fNtestFiles = other.fNtestFiles;
a3e84053 257 fNrunsPerMaster = other.fNrunsPerMaster;
258 fMaxMergeFiles = other.fMaxMergeFiles;
259 fNsubmitted = other.fNsubmitted;
260 fProductionMode = other.fProductionMode;
cd11251e 261 fOutputToRunNo = other.fOutputToRunNo;
0f389141 262 fMergeViaJDL = other.fMergeViaJDL;
263 fFastReadOption = other.fFastReadOption;
264 fOverwriteMode = other.fOverwriteMode;
c57f56b7 265 fRunNumbers = other.fRunNumbers;
266 fExecutable = other.fExecutable;
0a1c1f7f 267 fExecutableCommand = other.fExecutableCommand;
c57f56b7 268 fArguments = other.fArguments;
631c0b05 269 fExecutableArgs = other.fExecutableArgs;
c57f56b7 270 fAnalysisMacro = other.fAnalysisMacro;
271 fAnalysisSource = other.fAnalysisSource;
d5c6455a 272 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 273 fAdditionalLibs = other.fAdditionalLibs;
274 fSplitMode = other.fSplitMode;
275 fAPIVersion = other.fAPIVersion;
276 fROOTVersion = other.fROOTVersion;
277 fAliROOTVersion = other.fAliROOTVersion;
648174cf 278 fExternalPackages = other.fExternalPackages;
c57f56b7 279 fUser = other.fUser;
280 fGridWorkingDir = other.fGridWorkingDir;
281 fGridDataDir = other.fGridDataDir;
282 fDataPattern = other.fDataPattern;
283 fGridOutputDir = other.fGridOutputDir;
284 fOutputArchive = other.fOutputArchive;
285 fOutputFiles = other.fOutputFiles;
286 fInputFormat = other.fInputFormat;
e7c71df0 287 fDatasetName = other.fDatasetName;
c57f56b7 288 fJDLName = other.fJDLName;
bb885a9e 289 fMergeExcludes = other.fMergeExcludes;
f965131e 290 fIncludePath = other.fIncludePath;
bb885a9e 291 fCloseSE = other.fCloseSE;
0df6ccf2 292 fFriendChainName = other.fFriendChainName;
c6cb3634 293 fJobTag = other.fJobTag;
648174cf 294 fOutputSingle = other.fOutputSingle;
5fce53f4 295 fRunPrefix = other.fRunPrefix;
c57f56b7 296 if (other.fInputFiles) {
297 fInputFiles = new TObjArray();
298 TIter next(other.fInputFiles);
299 TObject *obj;
300 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
301 fInputFiles->SetOwner();
302 }
4e5c5506 303 if (other.fPackages) {
304 fPackages = new TObjArray();
305 TIter next(other.fPackages);
306 TObject *obj;
307 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
308 fPackages->SetOwner();
309 }
c57f56b7 310 }
311 return *this;
312}
313
f965131e 314//______________________________________________________________________________
315void AliAnalysisAlien::AddIncludePath(const char *path)
316{
317// Add include path in the remote analysis macro.
318 TString p(path);
319 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
320 else fIncludePath += Form("-I%s ", path);
321}
322
c57f56b7 323//______________________________________________________________________________
324void AliAnalysisAlien::AddRunNumber(Int_t run)
325{
326// Add a run number to the list of runs to be processed.
327 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 328 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 329}
330
ee75cfc3 331//______________________________________________________________________________
332void AliAnalysisAlien::AddRunNumber(const char* run)
333{
334// Add a run number to the list of runs to be processed.
335 if (fRunNumbers.Length()) fRunNumbers += " ";
336 fRunNumbers += run;
337}
338
c57f56b7 339//______________________________________________________________________________
340void AliAnalysisAlien::AddDataFile(const char *lfn)
341{
342// Adds a data file to the input to be analysed. The file should be a valid LFN
343// or point to an existing file in the alien workdir.
344 if (!fInputFiles) fInputFiles = new TObjArray();
345 fInputFiles->Add(new TObjString(lfn));
346}
648174cf 347
348//______________________________________________________________________________
349void AliAnalysisAlien::AddExternalPackage(const char *package)
350{
351// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
352 if (fExternalPackages) fExternalPackages += " ";
353 fExternalPackages += package;
354}
355
c57f56b7 356//______________________________________________________________________________
357Bool_t AliAnalysisAlien::Connect()
358{
359// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
360 if (gGrid && gGrid->IsConnected()) return kTRUE;
c57f56b7 361 if (!gGrid) {
362 Info("Connect", "Trying to connect to AliEn ...");
363 TGrid::Connect("alien://");
364 }
365 if (!gGrid || !gGrid->IsConnected()) {
366 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
367 return kFALSE;
368 }
369 fUser = gGrid->GetUser();
370 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
371 return kTRUE;
372}
373
374//______________________________________________________________________________
375void AliAnalysisAlien::CdWork()
376{
377// Check validity of alien workspace. Create directory if possible.
378 if (!Connect()) {
379 Error("CdWork", "Alien connection required");
380 return;
381 }
382 TString homedir = gGrid->GetHomeDirectory();
383 TString workdir = homedir + fGridWorkingDir;
923e2ca5 384 if (DirectoryExists(workdir)) {
385 gGrid->Cd(workdir);
386 return;
387 }
388 // Work directory not existing - create it
389 gGrid->Cd(homedir);
390 if (gGrid->Mkdir(workdir)) {
391 gGrid->Cd(fGridWorkingDir);
392 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
393 } else {
394 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
395 workdir.Data(), homedir.Data());
396 fGridWorkingDir = "";
397 }
c57f56b7 398}
399
348be253 400//______________________________________________________________________________
401Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
402{
403// Check if file copying is possible.
404 if (!Connect()) {
405 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
406 return kFALSE;
407 }
d3339be3 408 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
409 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
348be253 410 // Check if alien_CLOSE_SE is defined
411 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
412 if (!closeSE.IsNull()) {
413 Info("CheckFileCopy", "Your current close storage is pointing to: \
414 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
415 } else {
416 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
417 }
418 // Check if grid directory exists.
419 if (!DirectoryExists(alienpath)) {
420 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
421 return kFALSE;
422 }
d3339be3 423 TFile f("plugin_test_copy", "RECREATE");
348be253 424 // User may not have write permissions to current directory
425 if (f.IsZombie()) {
426 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
427 gSystem->WorkingDirectory());
428 return kFALSE;
429 }
430 f.Close();
d3339be3 431 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
432 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
433 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
348be253 434 \n# 1. Make sure you have write permissions there. If this is the case: \
435 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
436 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
437 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
438 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
439 gSystem->Unlink(f.GetName());
440 return kFALSE;
441 }
442 gSystem->Unlink(f.GetName());
d3339be3 443 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
444 Info("CheckFileCopy", "### ...SUCCESS ###");
348be253 445 return kTRUE;
446}
447
c57f56b7 448//______________________________________________________________________________
449Bool_t AliAnalysisAlien::CheckInputData()
450{
451// Check validity of input data. If necessary, create xml files.
a8739e8a 452 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 453 if (!fGridDataDir.Length()) {
454 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
455 return kFALSE;
456 }
457 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
458 return kTRUE;
c57f56b7 459 }
460 // Process declared files
461 Bool_t is_collection = kFALSE;
462 Bool_t is_xml = kFALSE;
463 Bool_t use_tags = kFALSE;
464 Bool_t checked = kFALSE;
465 CdWork();
466 TString file;
467 TString workdir = gGrid->GetHomeDirectory();
468 workdir += fGridWorkingDir;
469 if (fInputFiles) {
470 TObjString *objstr;
471 TIter next(fInputFiles);
472 while ((objstr=(TObjString*)next())) {
473 file = workdir;
474 file += "/";
475 file += objstr->GetString();
476 // Store full lfn path
477 if (FileExists(file)) objstr->SetString(file);
478 else {
479 file = objstr->GetName();
480 if (!FileExists(objstr->GetName())) {
481 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
482 objstr->GetName(), workdir.Data());
483 return kFALSE;
484 }
485 }
486 Bool_t iscoll, isxml, usetags;
487 CheckDataType(file, iscoll, isxml, usetags);
488 if (!checked) {
489 checked = kTRUE;
490 is_collection = iscoll;
491 is_xml = isxml;
492 use_tags = usetags;
493 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
494 } else {
495 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
496 Error("CheckInputData", "Some conflict was found in the types of inputs");
497 return kFALSE;
498 }
499 }
500 }
501 }
502 // Process requested run numbers
a8739e8a 503 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 504 // Check validity of alien data directory
505 if (!fGridDataDir.Length()) {
506 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
507 return kFALSE;
508 }
923e2ca5 509 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 510 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
511 return kFALSE;
512 }
513 if (is_collection) {
514 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
515 return kFALSE;
516 }
517
518 if (checked && !is_xml) {
519 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
520 return kFALSE;
521 }
522 // Check validity of run number(s)
523 TObjArray *arr;
524 TObjString *os;
319593fb 525 Int_t nruns = 0;
904f9f5f 526 TString schunk, schunk2;
c57f56b7 527 TString path;
528 if (!checked) {
529 checked = kTRUE;
530 use_tags = fDataPattern.Contains("tag");
531 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
532 }
533 if (use_tags != fDataPattern.Contains("tag")) {
534 Error("CheckInputData", "Cannot mix input files using/not using tags");
535 return kFALSE;
536 }
537 if (fRunNumbers.Length()) {
a8739e8a 538 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 539 arr = fRunNumbers.Tokenize(" ");
540 TIter next(arr);
541 while ((os=(TObjString*)next())) {
542 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 543 if (!DirectoryExists(path)) {
544 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 545 continue;
c57f56b7 546 }
547 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
548 TString msg = "\n##### file: ";
549 msg += path;
550 msg += " type: xml_collection;";
551 if (use_tags) msg += " using_tags: Yes";
552 else msg += " using_tags: No";
553 Info("CheckDataType", msg.Data());
319593fb 554 if (fNrunsPerMaster<2) {
d2a409b2 555 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 556 } else {
557 nruns++;
558 if (((nruns-1)%fNrunsPerMaster) == 0) {
559 schunk = os->GetString();
560 }
561 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
562 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 563 AddDataFile(schunk);
319593fb 564 }
c57f56b7 565 }
566 delete arr;
a8739e8a 567 } else {
568 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
569 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 570 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 571 if (!DirectoryExists(path)) {
572// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 573 continue;
574 }
5fce53f4 575 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 576 TString msg = "\n##### file: ";
577 msg += path;
578 msg += " type: xml_collection;";
579 if (use_tags) msg += " using_tags: Yes";
580 else msg += " using_tags: No";
581 Info("CheckDataType", msg.Data());
319593fb 582 if (fNrunsPerMaster<2) {
5fce53f4 583 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 584 } else {
585 nruns++;
586 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 587 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 588 }
904f9f5f 589 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 590 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 591 schunk += schunk2;
d2a409b2 592 AddDataFile(schunk);
319593fb 593 }
a8739e8a 594 }
904f9f5f 595 if (!fInputFiles) {
596 schunk += schunk2;
597 AddDataFile(schunk);
598 }
c57f56b7 599 }
600 return kTRUE;
601}
602
603//______________________________________________________________________________
604Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
605{
606// Create dataset for the grid data directory + run number.
607 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
608 if (!Connect()) {
609 Error("CreateDataset", "Cannot create dataset with no grid connection");
610 return kFALSE;
611 }
612
613 // Cd workspace
614 CdWork();
615 TString workdir = gGrid->GetHomeDirectory();
616 workdir += fGridWorkingDir;
617
618 // Compose the 'find' command arguments
619 TString command;
620 TString options = "-x collection ";
bb885a9e 621 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 622 TString conditions = "";
623
624 TString file;
625 TString path;
319593fb 626 Int_t nruns = 0;
904f9f5f 627 TString schunk, schunk2;
ab254fd1 628 TGridCollection *cbase=0, *cadd=0;
d2a409b2 629 if (!fRunNumbers.Length() && !fRunRange[0]) {
630 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
631 // Make a single data collection from data directory.
632 path = fGridDataDir;
923e2ca5 633 if (!DirectoryExists(path)) {
d2a409b2 634 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
635 return kFALSE;
636 }
923e2ca5 637// CdWork();
d2a409b2 638 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
639 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 640 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 641 command = "find ";
642 command += options;
643 command += path;
644 command += " ";
645 command += pattern;
646 command += conditions;
84fcd93f 647 printf("command: %s\n", command.Data());
d2a409b2 648 TGridResult *res = gGrid->Command(command);
649 if (res) delete res;
650 // Write standard output to file
651 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
d3339be3 652 Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
653 Bool_t null_file = kFALSE;
654 if (!has_grep) {
655 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
656 } else {
657 null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
658 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
659 return kFALSE;
660 }
0f389141 661 }
662 Bool_t fileExists = FileExists(file);
663 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 664 // Copy xml file to alien space
0f389141 665 if (fileExists) gGrid->Rm(file);
d2a409b2 666 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
667 if (!FileExists(file)) {
668 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
669 return kFALSE;
670 }
671 // Update list of files to be processed.
672 }
673 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
674 return kTRUE;
675 }
c57f56b7 676 // Several runs
d3339be3 677 Bool_t null_result = kTRUE;
a8739e8a 678 if (fRunNumbers.Length()) {
679 TObjArray *arr = fRunNumbers.Tokenize(" ");
680 TObjString *os;
681 TIter next(arr);
682 while ((os=(TObjString*)next())) {
683 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 684 if (!DirectoryExists(path)) continue;
685// CdWork();
a8739e8a 686 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
687 else file = Form("%s.xml", os->GetString().Data());
319593fb 688 // If local collection file does not exist, create it via 'find' command.
d3339be3 689 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 690 command = "find ";
691 command += options;
692 command += path;
693 command += pattern;
694 command += conditions;
695 TGridResult *res = gGrid->Command(command);
696 if (res) delete res;
697 // Write standard output to file
698 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
d3339be3 699 Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
700 Bool_t null_file = kFALSE;
701 if (!has_grep) {
702 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
703 } else {
704 null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
705 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
706 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
707 continue;
708 }
709 null_result = kFALSE;
710 }
a8739e8a 711 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 712 // Check if there is one run per master job.
713 if (fNrunsPerMaster<2) {
714 if (FileExists(file)) {
0f389141 715 if (fOverwriteMode) gGrid->Rm(file);
716 else {
717 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
718 continue;
719 }
319593fb 720 }
721 // Copy xml file to alien space
722 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
723 if (!FileExists(file)) {
724 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
725 delete arr;
726 return kFALSE;
727 }
728 } else {
729 nruns++;
730 if (((nruns-1)%fNrunsPerMaster) == 0) {
731 schunk = os->GetString();
732 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
733 } else {
734 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 735 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 736 cbase->Add(cadd);
737 delete cadd;
738 }
739 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
740 continue;
741 }
742 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 743 if (FileExists(schunk)) {
744 if (fOverwriteMode) gGrid->Rm(file);
745 else {
746 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
747 continue;
748 }
319593fb 749 }
84fcd93f 750 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 751 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 752 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 753 if (!FileExists(schunk)) {
754 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
755 delete arr;
756 return kFALSE;
757 }
d3339be3 758 }
a8739e8a 759 }
760 delete arr;
d3339be3 761 if (null_result) {
762 Error("CreateDataset", "No valid dataset corresponding to the query!");
763 return kFALSE;
764 }
a8739e8a 765 } else {
766 // Process a full run range.
767 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 768 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 769 if (!DirectoryExists(path)) continue;
770// CdWork();
a8739e8a 771 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 772 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
0f389141 773 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
774 if (fOverwriteMode) gGrid->Rm(file);
775 else {
776 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
777 continue;
778 }
a8739e8a 779 }
319593fb 780 // If local collection file does not exist, create it via 'find' command.
d3339be3 781 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
319593fb 782 command = "find ";
783 command += options;
784 command += path;
785 command += pattern;
786 command += conditions;
787 TGridResult *res = gGrid->Command(command);
788 if (res) delete res;
789 // Write standard output to file
790 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
d3339be3 791 Bool_t has_grep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
792 Bool_t null_file = kFALSE;
793 if (!has_grep) {
794 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
795 } else {
796 null_file = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
797 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
798 continue;
799 }
800 null_result = kFALSE;
319593fb 801 }
a8739e8a 802 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 803 // Check if there is one run per master job.
804 if (fNrunsPerMaster<2) {
805 if (FileExists(file)) {
0f389141 806 if (fOverwriteMode) gGrid->Rm(file);
807 else {
808 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
809 continue;
810 }
319593fb 811 }
812 // Copy xml file to alien space
813 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
814 if (!FileExists(file)) {
815 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
816 return kFALSE;
817 }
818 } else {
819 nruns++;
95e5b448 820 // Check if the collection for the chunk exist locally.
821 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 822 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
823 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
824 else continue;
825 }
84fcd93f 826 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 827 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 828 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 829 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
830 } else {
831 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
832 cbase->Add(cadd);
833 delete cadd;
834 }
904f9f5f 835 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
836 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 837 continue;
838 }
904f9f5f 839 schunk = schunk2;
319593fb 840 if (FileExists(schunk)) {
0f389141 841 if (fOverwriteMode) gGrid->Rm(schunk);
842 else {
843 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
844 continue;
845 }
319593fb 846 }
84fcd93f 847 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 848 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 849 if (FileExists(schunk)) {
0f389141 850 if (fOverwriteMode) gGrid->Rm(schunk);
851 else {
852 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
853 continue;
854 }
95e5b448 855 }
319593fb 856 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
857 if (!FileExists(schunk)) {
858 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
859 return kFALSE;
860 }
861 }
c57f56b7 862 }
d3339be3 863 if (null_result) {
864 Error("CreateDataset", "No valid dataset corresponding to the query!");
865 return kFALSE;
866 }
a8739e8a 867 }
c57f56b7 868 return kTRUE;
869}
870
871//______________________________________________________________________________
872Bool_t AliAnalysisAlien::CreateJDL()
873{
874// Generate a JDL file according to current settings. The name of the file is
875// specified by fJDLName.
876 Bool_t error = kFALSE;
877 TObjArray *arr = 0;
878 Bool_t copy = kTRUE;
879 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
880 Bool_t generate = kTRUE;
881 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
882 if (!Connect()) {
883 Error("CreateJDL", "Alien connection required");
884 return kFALSE;
885 }
886 // Check validity of alien workspace
887 CdWork();
888 TString workdir = gGrid->GetHomeDirectory();
889 workdir += fGridWorkingDir;
890 if (generate) {
891 TObjString *os;
892 if (!fInputFiles) {
893 Error("CreateJDL()", "Define some input files for your analysis.");
894 error = kTRUE;
895 }
896 // Compose list of input files
897 // Check if output files were defined
898 if (!fOutputFiles.Length()) {
899 Error("CreateJDL", "You must define at least one output file");
900 error = kTRUE;
901 }
902 // Check if an output directory was defined and valid
903 if (!fGridOutputDir.Length()) {
904 Error("CreateJDL", "You must define AliEn output directory");
905 error = kTRUE;
906 } else {
d2a409b2 907 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 908 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 909 if (gGrid->Mkdir(fGridOutputDir)) {
910 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
911 } else {
912 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 913 // error = kTRUE;
c57f56b7 914 }
915 }
916 gGrid->Cd(workdir);
917 }
918 // Exit if any error up to now
919 if (error) return kFALSE;
920 // Set JDL fields
0f389141 921 if (!fUser.IsNull()) {
922 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
923 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
924 }
925 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
926 TString mergeExec = fExecutable;
927 mergeExec.ReplaceAll(".sh", "_merge.sh");
928 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
929 mergeExec.ReplaceAll(".sh", ".C");
930 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 931 if (!fArguments.IsNull())
932 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
1f0d1ca2 933 fMergingJDL->SetArguments("$1");
934 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
935 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
936 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
937 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
938
0f389141 939 if (fMaxInitFailed > 0) {
c57f56b7 940 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 941 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
942 }
943 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 944 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 945 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
946 }
947 if (fSplitMode.Length()) {
c57f56b7 948 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 949 fGridJDL->SetDescription("Split", "We split per SE or file");
950 }
951 if (!fAliROOTVersion.IsNull()) {
952 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
953 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
954 }
955 if (!fROOTVersion.IsNull()) {
c57f56b7 956 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 957 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
958 }
959 if (!fAPIVersion.IsNull()) {
c57f56b7 960 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 961 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
962 }
648174cf 963 if (!fExternalPackages.IsNull()) {
964 arr = fExternalPackages.Tokenize(" ");
965 TIter next(arr);
966 while ((os=(TObjString*)next())) {
967 TString pkgname = os->GetString();
968 Int_t index = pkgname.Index("::");
969 TString pkgversion = pkgname(index+2, pkgname.Length());
970 pkgname.Remove(index);
971 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 972 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 973 }
974 delete arr;
975 }
0f389141 976 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
977 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
978 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 979 TString analysisFile = fExecutable;
980 analysisFile.ReplaceAll(".sh", ".root");
981 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 982 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 983 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
984 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
985 if (fAdditionalLibs.Length()) {
986 arr = fAdditionalLibs.Tokenize(" ");
987 TIter next(arr);
988 while ((os=(TObjString*)next())) {
989 if (os->GetString().Contains(".so")) continue;
990 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 991 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 992 }
993 delete arr;
994 }
4e5c5506 995 if (fPackages) {
996 TIter next(fPackages);
997 TObject *obj;
0f389141 998 while ((obj=next())) {
4e5c5506 999 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 1000 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1001 }
4e5c5506 1002 }
c57f56b7 1003 if (fOutputArchive.Length()) {
1004 arr = fOutputArchive.Tokenize(" ");
1005 TIter next(arr);
0f389141 1006 Bool_t first = kTRUE;
1007 const char *comment = "Files to be archived";
1008 const char *comment1 = comment;
1009 while ((os=(TObjString*)next())) {
1010 if (!first) comment = NULL;
1011 if (!os->GetString().Contains("@") && fCloseSE.Length())
1012 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1013 else
1014 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1015 first = kFALSE;
1016 }
c57f56b7 1017 delete arr;
0f389141 1018 TString outputArchive = fOutputArchive;
1019 if (!fMergeExcludes.IsNull()) {
1020 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1021 TIter next1(arr);
1022 while ((os=(TObjString*)next1())) {
0f389141 1023 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1024 outputArchive.ReplaceAll(os->GetString(),"");
1025 }
1026 delete arr;
1027 }
1028 arr = outputArchive.Tokenize(" ");
705adb3e 1029 TIter next2(arr);
0f389141 1030 comment = comment1;
1031 first = kTRUE;
705adb3e 1032 while ((os=(TObjString*)next2())) {
0f389141 1033 if (!first) comment = NULL;
1034 if (!os->GetString().Contains("@") && fCloseSE.Length())
1035 fMergingJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1036 else
1037 fMergingJDL->AddToOutputArchive(os->GetString(), comment);
1038 first = kFALSE;
1039 }
1040 delete arr;
c57f56b7 1041 }
c57f56b7 1042 arr = fOutputFiles.Tokenize(" ");
1043 TIter next(arr);
0f389141 1044 Bool_t first = kTRUE;
1045 const char *comment = "Files to be archived";
1046 const char *comment1 = comment;
43da816a 1047 while ((os=(TObjString*)next())) {
e1eaf596 1048 // Ignore ouputs in jdl that are also in outputarchive
1049 TString sout = os->GetString();
1050 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1051 if (fOutputArchive.Contains(sout)) continue;
0f389141 1052 if (!first) comment = NULL;
43da816a 1053 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 1054 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 1055 else
0f389141 1056 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
1057 first = kFALSE;
43da816a 1058 }
c57f56b7 1059 delete arr;
0f389141 1060 if (fOutputFiles.Length()) {
1061 TString outputFiles = fOutputFiles;
1062 if (!fMergeExcludes.IsNull()) {
1063 arr = fMergeExcludes.Tokenize(" ");
705adb3e 1064 TIter next1(arr);
1065 while ((os=(TObjString*)next1())) {
0f389141 1066 outputFiles.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1067 outputFiles.ReplaceAll(os->GetString(),"");
1068 }
1069 delete arr;
1070 }
1071 arr = outputFiles.Tokenize(" ");
705adb3e 1072 TIter next2(arr);
0f389141 1073 comment = comment1;
1074 first = kTRUE;
705adb3e 1075 while ((os=(TObjString*)next2())) {
0f389141 1076 // Ignore ouputs in jdl that are also in outputarchive
1077 TString sout = os->GetString();
1078 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1079 if (fOutputArchive.Contains(sout)) continue;
1080 if (!first) comment = NULL;
1081 if (!os->GetString().Contains("@") && fCloseSE.Length())
1082 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1083 else
1084 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
1085 }
1086 delete arr;
1087 }
1088 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1089 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
0d5d317c 1090 TString validationScript = fExecutable;
1091 validationScript.ReplaceAll(".sh", "_validation.sh");
0f389141 1092 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1093 validationScript = fExecutable;
1094 validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
1095 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1096 if (fMasterResubmitThreshold) {
1097 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1098 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1099 }
d2a409b2 1100 // Write a jdl with 2 input parameters: collection name and output dir name.
1101 WriteJDL(copy);
c57f56b7 1102 }
1103 // Copy jdl to grid workspace
a8739e8a 1104 if (copy) {
b5e4aaa7 1105 // Check if an output directory was defined and valid
1106 if (!fGridOutputDir.Length()) {
1107 Error("CreateJDL", "You must define AliEn output directory");
1108 return kFALSE;
1109 } else {
1110 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1111 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b5e4aaa7 1112 if (gGrid->Mkdir(fGridOutputDir)) {
1113 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1114 } else {
1115 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1116 return kFALSE;
1117 }
1118 }
1119 gGrid->Cd(workdir);
1120 }
648174cf 1121 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1122 TString mergeJDLName = fExecutable;
1123 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1124 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1125 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1126 if (fProductionMode) {
648174cf 1127 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1128 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1129 }
648174cf 1130 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1131 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1132 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1133 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1134 if (fMergeViaJDL) {
1135 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1136 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1137 }
648174cf 1138 }
c57f56b7 1139 if (fAdditionalLibs.Length()) {
e7c71df0 1140 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1141 TObjString *os;
1142 TIter next(arr);
1143 while ((os=(TObjString*)next())) {
c57f56b7 1144 if (os->GetString().Contains(".so")) continue;
4e5c5506 1145 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1146 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1147 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1148 }
1149 delete arr;
1150 }
4e5c5506 1151 if (fPackages) {
1152 TIter next(fPackages);
1153 TObject *obj;
1154 while ((obj=next())) {
fdbbc7be 1155 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1156 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1157 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1158 }
1159 }
c57f56b7 1160 }
1161 return kTRUE;
1162}
1163
a8739e8a 1164//______________________________________________________________________________
d2a409b2 1165Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1166{
1167// Writes one or more JDL's corresponding to findex. If findex is negative,
1168// all run numbers are considered in one go (jdl). For non-negative indices
1169// they correspond to the indices in the array fInputFiles.
1170 if (!fInputFiles) return kFALSE;
1171 TObjString *os;
a8739e8a 1172 TString workdir = gGrid->GetHomeDirectory();
1173 workdir += fGridWorkingDir;
d2a409b2 1174
1175 if (!fRunNumbers.Length() && !fRunRange[0]) {
1176 // One jdl with no parameters in case input data is specified by name.
a8739e8a 1177 TIter next(fInputFiles);
1178 while ((os=(TObjString*)next()))
0f389141 1179 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()), "Input xml collections");
648174cf 1180 if (!fOutputSingle.IsNull())
0f389141 1181 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1182 else {
1183 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1184 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1185 }
a8739e8a 1186 } else {
d2a409b2 1187 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
0f389141 1188 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
cd11251e 1189 if (!fOutputSingle.IsNull()) {
0f389141 1190 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1191 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
cd11251e 1192 } else {
0f389141 1193 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1194 fMergingJDL->SetOutputDirectory(Form("%s/$1", fGridOutputDir.Data()), "Output directory");
cd11251e 1195 }
a8739e8a 1196 }
1197
1198
1199 // Generate the JDL as a string
1200 TString sjdl = fGridJDL->Generate();
0f389141 1201 TString sjdl1 = fMergingJDL->Generate();
a8739e8a 1202 Int_t index;
a8739e8a 1203 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1204 sjdl.ReplaceAll("(member", "\n (member");
1205 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1206 sjdl.ReplaceAll("{", "{\n ");
1207 sjdl.ReplaceAll("};", "\n};");
1208 sjdl.ReplaceAll("{\n \n", "{\n");
1209 sjdl.ReplaceAll("\n\n", "\n");
1210 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1211 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1212 sjdl1.ReplaceAll("(member", "\n (member");
1213 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1214 sjdl1.ReplaceAll("{", "{\n ");
1215 sjdl1.ReplaceAll("};", "\n};");
1216 sjdl1.ReplaceAll("{\n \n", "{\n");
1217 sjdl1.ReplaceAll("\n\n", "\n");
1218 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1219 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1220 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1221 index = sjdl.Index("JDLVariables");
1222 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
0f389141 1223 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1224 sjdl1.Prepend(Form("Jobtag = {\n \"comment:Merging_%s\"\n};\n", fJobTag.Data()));
1225 index = sjdl1.Index("JDLVariables");
1226 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
a8739e8a 1227 // Write jdl to file
a8739e8a 1228 ofstream out;
d2a409b2 1229 out.open(fJDLName.Data(), ios::out);
a8739e8a 1230 if (out.bad()) {
d2a409b2 1231 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1232 return kFALSE;
1233 }
1234 out << sjdl << endl;
0f389141 1235 TString mergeJDLName = fExecutable;
1236 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1237 if (fMergeViaJDL) {
1238 ofstream out1;
1239 out1.open(mergeJDLName.Data(), ios::out);
1240 if (out.bad()) {
1241 Error("CreateJDL", "Bad file name: %s", mergeJDLName.Data());
1242 return kFALSE;
1243 }
1244 out1 << sjdl1 << endl;
1245 }
a8739e8a 1246
1247 // Copy jdl to grid workspace
1248 if (!copy) {
1249 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1250 } else {
d2a409b2 1251 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1252 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1253 if (fProductionMode) {
b5fe9cba 1254 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1255 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1256 }
d2a409b2 1257 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1258 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1259 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1260 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1261 if (fMergeViaJDL) {
1262 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1263 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1264 }
a8739e8a 1265 }
1266 return kTRUE;
1267}
1268
c57f56b7 1269//______________________________________________________________________________
5513444a 1270Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1271{
1272// Returns true if file exists.
5513444a 1273 if (!gGrid) return kFALSE;
c57f56b7 1274 TGridResult *res = gGrid->Ls(lfn);
1275 if (!res) return kFALSE;
1276 TMap *map = dynamic_cast<TMap*>(res->At(0));
1277 if (!map) {
1278 delete res;
1279 return kFALSE;
1280 }
1281 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1282 if (!objs || !objs->GetString().Length()) {
1283 delete res;
1284 return kFALSE;
1285 }
1286 delete res;
1287 return kTRUE;
1288}
1289
923e2ca5 1290//______________________________________________________________________________
1291Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1292{
1293// Returns true if directory exists. Can be also a path.
1294 if (!gGrid) return kFALSE;
1295 // Check if dirname is a path
1296 TString dirstripped = dirname;
1297 dirstripped = dirstripped.Strip();
1298 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1299 TString dir = gSystem->BaseName(dirstripped);
1300 dir += "/";
1301 TString path = gSystem->DirName(dirstripped);
1302 TGridResult *res = gGrid->Ls(path, "-F");
1303 if (!res) return kFALSE;
1304 TIter next(res);
1305 TMap *map;
1306 TObject *obj;
1307 while ((map=dynamic_cast<TMap*>(next()))) {
1308 obj = map->GetValue("name");
1309 if (!obj) break;
1310 if (dir == obj->GetName()) {
1311 delete res;
1312 return kTRUE;
1313 }
1314 }
1315 delete res;
1316 return kFALSE;
1317}
1318
c57f56b7 1319//______________________________________________________________________________
1320void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1321{
1322// Check input data type.
1323 is_collection = kFALSE;
1324 is_xml = kFALSE;
1325 use_tags = kFALSE;
1326 if (!gGrid) {
1327 Error("CheckDataType", "No connection to grid");
1328 return;
1329 }
1330 is_collection = IsCollection(lfn);
1331 TString msg = "\n##### file: ";
1332 msg += lfn;
1333 if (is_collection) {
1334 msg += " type: raw_collection;";
1335 // special treatment for collections
1336 is_xml = kFALSE;
1337 // check for tag files in the collection
1338 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1339 if (!res) {
1340 msg += " using_tags: No (unknown)";
1341 Info("CheckDataType", msg.Data());
1342 return;
1343 }
1344 const char* typeStr = res->GetKey(0, "origLFN");
1345 if (!typeStr || !strlen(typeStr)) {
1346 msg += " using_tags: No (unknown)";
1347 Info("CheckDataType", msg.Data());
1348 return;
1349 }
1350 TString file = typeStr;
1351 use_tags = file.Contains(".tag");
1352 if (use_tags) msg += " using_tags: Yes";
1353 else msg += " using_tags: No";
1354 Info("CheckDataType", msg.Data());
1355 return;
1356 }
1357 TString slfn(lfn);
1358 slfn.ToLower();
1359 is_xml = slfn.Contains(".xml");
1360 if (is_xml) {
1361 // Open xml collection and check if there are tag files inside
1362 msg += " type: xml_collection;";
1363 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1364 if (!coll) {
1365 msg += " using_tags: No (unknown)";
1366 Info("CheckDataType", msg.Data());
1367 return;
1368 }
1369 TMap *map = coll->Next();
1370 if (!map) {
1371 msg += " using_tags: No (unknown)";
1372 Info("CheckDataType", msg.Data());
1373 return;
1374 }
1375 map = (TMap*)map->GetValue("");
1376 TString file;
1377 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1378 use_tags = file.Contains(".tag");
1379 delete coll;
1380 if (use_tags) msg += " using_tags: Yes";
1381 else msg += " using_tags: No";
1382 Info("CheckDataType", msg.Data());
1383 return;
1384 }
1385 use_tags = slfn.Contains(".tag");
1386 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1387 else msg += " type: unknown file;";
c57f56b7 1388 if (use_tags) msg += " using_tags: Yes";
1389 else msg += " using_tags: No";
1390 Info("CheckDataType", msg.Data());
1391}
1392
4e5c5506 1393//______________________________________________________________________________
1394void AliAnalysisAlien::EnablePackage(const char *package)
1395{
1396// Enables a par file supposed to exist in the current directory.
1397 TString pkg(package);
1398 pkg.ReplaceAll(".par", "");
1399 pkg += ".par";
1400 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1401 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1402 return;
1403 }
1404 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1405 Info("EnablePackage", "AliEn plugin will use .par packages");
1406 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1407 if (!fPackages) {
1408 fPackages = new TObjArray();
1409 fPackages->SetOwner();
1410 }
1411 fPackages->Add(new TObjString(pkg));
1412}
1413
d2a409b2 1414//______________________________________________________________________________
1415const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1416{
1417// Get job status for all jobs with jobid>jobidstart.
1418 static char mstatus[20];
1419 mstatus[0] = '\0';
1420 nrunning = 0;
1421 nwaiting = 0;
1422 nerror = 0;
1423 ndone = 0;
1424 TGridJobStatusList *list = gGrid->Ps("");
1425 if (!list) return mstatus;
1426 Int_t nentries = list->GetSize();
1427 TGridJobStatus *status;
1428 Int_t pid;
1429 for (Int_t ijob=0; ijob<nentries; ijob++) {
1430 status = (TGridJobStatus *)list->At(ijob);
1431 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1432 if (pid<jobidstart) continue;
1433 if (pid == lastid) {
1434 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1435 }
1436 switch (status->GetStatus()) {
1437 case TGridJobStatus::kWAITING:
1438 nwaiting++; break;
1439 case TGridJobStatus::kRUNNING:
1440 nrunning++; break;
1441 case TGridJobStatus::kABORTED:
1442 case TGridJobStatus::kFAIL:
1443 case TGridJobStatus::kUNKNOWN:
1444 nerror++; break;
1445 case TGridJobStatus::kDONE:
1446 ndone++;
1447 }
1448 }
1449 list->Delete();
1450 delete list;
1451 return mstatus;
1452}
1453
c57f56b7 1454//______________________________________________________________________________
1455Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1456{
1457// Returns true if file is a collection. Functionality duplicated from
1458// TAlien::Type() because we don't want to directly depend on TAlien.
1459 if (!gGrid) {
1460 Error("IsCollection", "No connection to grid");
1461 return kFALSE;
1462 }
1463 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1464 if (!res) return kFALSE;
1465 const char* typeStr = res->GetKey(0, "type");
1466 if (!typeStr || !strlen(typeStr)) return kFALSE;
1467 if (!strcmp(typeStr, "collection")) return kTRUE;
1468 delete res;
1469 return kFALSE;
1470}
1471
fe2d7fc2 1472//______________________________________________________________________________
1473Bool_t AliAnalysisAlien::IsSingleOutput() const
1474{
1475// Check if single-ouput option is on.
1476 return (!fOutputSingle.IsNull());
1477}
1478
16a4353c 1479//______________________________________________________________________________
1480void AliAnalysisAlien::Print(Option_t *) const
1481{
1482// Print current plugin settings.
84fcd93f 1483 printf("### AliEn analysis plugin current settings ###\n");
348be253 1484 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1485 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
84fcd93f 1486 printf("= Production mode:______________________________ %d\n", fProductionMode);
1487 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1488 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1489 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1490 if (fUser.Length())
84fcd93f 1491 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1492 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1493 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1494 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1495 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1496 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1497 if (fRunNumbers.Length())
84fcd93f 1498 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1499 if (fRunRange[0])
5fce53f4 1500 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1501 if (!fRunRange[0] && !fRunNumbers.Length()) {
1502 TIter next(fInputFiles);
1503 TObject *obj;
1504 TString list;
1505 while ((obj=next())) list += obj->GetName();
84fcd93f 1506 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1507 }
1508 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1509 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1510 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1511 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1512 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1513 printf("=====================================================================\n");
1514 printf("= Job price: ___________________________________ %d\n", fPrice);
1515 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1516 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1517 if (fMaxInitFailed>0)
84fcd93f 1518 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1519 if (fMasterResubmitThreshold>0)
84fcd93f 1520 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1521 if (fNrunsPerMaster>0)
84fcd93f 1522 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1523 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1524 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1525 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1526 if (fArguments.Length())
84fcd93f 1527 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1528 if (fExecutableArgs.Length())
1529 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1530 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1531 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1532 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1533 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1534 if (fDatasetName)
84fcd93f 1535 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1536 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1537 if (fIncludePath.Data())
84fcd93f 1538 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1539 if (fCloseSE.Length())
84fcd93f 1540 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1541 if (fFriendChainName.Length())
84fcd93f 1542 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1543 if (fPackages) {
1544 TIter next(fPackages);
1545 TObject *obj;
1546 TString list;
1547 while ((obj=next())) list += obj->GetName();
84fcd93f 1548 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1549 }
1550}
1551
c57f56b7 1552//______________________________________________________________________________
1553void AliAnalysisAlien::SetDefaults()
1554{
1555// Set default values for everything. What cannot be filled will be left empty.
1556 if (fGridJDL) delete fGridJDL;
1557 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1558 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1559 fPrice = 1;
1560 fTTL = 30000;
1561 fSplitMaxInputFileNumber = 100;
1562 fMaxInitFailed = 0;
1563 fMasterResubmitThreshold = 0;
bb885a9e 1564 fNtestFiles = 10;
a8739e8a 1565 fRunRange[0] = 0;
1566 fRunRange[1] = 0;
319593fb 1567 fNrunsPerMaster = 1;
16a4353c 1568 fMaxMergeFiles = 100;
c57f56b7 1569 fRunNumbers = "";
1570 fExecutable = "analysis.sh";
0a1c1f7f 1571 fExecutableCommand = "root -b -q";
c57f56b7 1572 fArguments = "";
631c0b05 1573 fExecutableArgs = "";
c57f56b7 1574 fAnalysisMacro = "myAnalysis.C";
1575 fAnalysisSource = "";
1576 fAdditionalLibs = "";
1577 fSplitMode = "se";
1578 fAPIVersion = "";
1579 fROOTVersion = "";
1580 fAliROOTVersion = "";
1581 fUser = ""; // Your alien user name
1582 fGridWorkingDir = "";
1583 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1584 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1585 fFriendChainName = "";
c57f56b7 1586 fGridOutputDir = "output";
1587 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1588 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1589 fInputFormat = "xml-single";
1590 fJDLName = "analysis.jdl";
c6cb3634 1591 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1592 fMergeExcludes = "";
0f389141 1593 fMergeViaJDL = 0;
348be253 1594 SetUseCopy(kTRUE);
1595 SetCheckCopy(kTRUE);
c57f56b7 1596}
1597
0f389141 1598//______________________________________________________________________________
1599Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge)
1600{
1601// Merge all registered outputs from basedir.
1602 TString output_file = output;
1603 TString command;
1604 TString output_chunk;
1605 TString previous_chunk = "";
1606 Int_t count_chunk = 0;
1607 Int_t count_zero = nmaxmerge;
1608 Bool_t merged = kTRUE;
1609 Int_t index = output_file.Index("@");
1610 if (index > 0) output_file.Remove(index);
1611 command = Form("find %s/ *%s", basedir, output_file.Data());
1612 printf("command: %s\n", command.Data());
1613 TGridResult *res = gGrid->Command(command);
1614 if (!res) {
1615 printf("Error: No result for the find command\n");
1616 return kFALSE;
1617 }
1618
1619 TFileMerger *fm = 0;
1620 TIter nextmap(res);
1621 TMap *map = 0;
1622 // Check if there is a merge operation to resume
1623 output_chunk = output_file;
1624 output_chunk.ReplaceAll(".root", "_*.root");
1625 // Check for existent temporary merge files
1626 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1627 while (1) {
1628 // Skip as many input files as in a chunk
1629 for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
1630 if (!map) {
1631 ::Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1632 delete res;
1633 return kFALSE;
1634 }
1635 output_chunk = output_file;
1636 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1637 count_chunk++;
1638 if (gSystem->AccessPathName(output_chunk)) continue;
1639 // Merged file with chunks up to <count_chunk> found
1640 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1641 previous_chunk = output_chunk;
1642 break;
1643 }
1644 }
1645 count_zero = nmaxmerge;
1646
1647 while ((map=(TMap*)nextmap())) {
1648 // Loop 'find' results and get next LFN
1649 if (count_zero == nmaxmerge) {
1650 // First file in chunk - create file merger and add previous chunk if any.
1651 fm = new TFileMerger(kFALSE);
1652 fm->SetFastMethod(kTRUE);
1653 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1654 output_chunk = output_file;
1655 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1656 }
1657 // If last file found, put merged results in the output file
1658 if (map == res->Last()) output_chunk = output_file;
1659 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1660 if (!objs || !objs->GetString().Length()) {
1661 // Nothing found - skip this output
1662 delete res;
1663 delete fm;
1664 return kFALSE;
1665 }
1666 // Add file to be merged and decrement chunk counter.
1667 fm->AddFile(objs->GetString());
1668 count_zero--;
1669 if (count_zero==0 || map == res->Last()) {
1670 fm->OutputFile(output_chunk);
1671 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1672 // Nothing found - skip this output
1673 ::Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1674 delete res;
1675 delete fm;
1676 return kFALSE;
1677 }
1678 // Merge the outputs, then go to next chunk
1679 if (!fm->Merge()) {
1680 ::Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1681 delete res;
1682 delete fm;
1683 merged = kFALSE;
1684 return kFALSE;
1685 } else {
1686 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
1687 gSystem->Unlink(previous_chunk);
1688 }
1689 if (map == res->Last()) {
1690 delete res;
1691 delete fm;
1692 break;
1693 }
1694 count_chunk++;
1695 count_zero = nmaxmerge;
1696 previous_chunk = output_chunk;
1697 }
1698 }
1699 return merged;
1700}
1701
c57f56b7 1702//______________________________________________________________________________
1703Bool_t AliAnalysisAlien::MergeOutputs()
1704{
1705// Merge analysis outputs existing in the AliEn space.
1706 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1707 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1708 if (!Connect()) {
1709 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1710 return kFALSE;
0f389141 1711 }
c9e8f7fd 1712 if (fMergeViaJDL) {
1713 if (!TestBit(AliAnalysisGrid::kMerge)) {
1714 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
1715 return kFALSE;
1716 }
1717 if (fProductionMode) {
1718 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
1719 return kFALSE;
1720 }
0f389141 1721 Info("MergeOutputs", "Submitting merging JDL");
a03be957 1722 if (!SubmitMerging()) return kFALSE;
0f389141 1723 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
c9e8f7fd 1724 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
001cb79e 1725 return kFALSE;
c57f56b7 1726 }
1727 // Get the output path
d2a409b2 1728 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1729 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1730 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1731 return kFALSE;
1732 }
1733 if (!fOutputFiles.Length()) {
1734 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1735 return kFALSE;
0f389141 1736 }
1737 // Check if fast read option was requested
1738 if (fFastReadOption) {
d3339be3 1739 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
1740 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
1741 gEnv->SetValue("XNet.ConnectTimeout",10);
1742 gEnv->SetValue("XNet.RequestTimeout",10);
0f389141 1743 gEnv->SetValue("XNet.MaxRedirectCount",2);
d3339be3 1744 gEnv->SetValue("XNet.ReconnectTimeout",10);
0f389141 1745 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 1746 }
1747 TObjArray *list = fOutputFiles.Tokenize(" ");
1748 TIter next(list);
1749 TObjString *str;
c57f56b7 1750 TString output_file;
1751 Bool_t merged = kTRUE;
1752 while((str=(TObjString*)next())) {
1753 output_file = str->GetString();
1754 Int_t index = output_file.Index("@");
1755 if (index > 0) output_file.Remove(index);
319593fb 1756 // Skip already merged outputs
1757 if (!gSystem->AccessPathName(output_file)) {
1758 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1759 continue;
1760 }
bb885a9e 1761 if (fMergeExcludes.Length() &&
1762 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1763 // Perform a 'find' command in the output directory, looking for registered outputs
0f389141 1764 merged = MergeOutput(output_file, fGridOutputDir, fMaxMergeFiles);
1765 if (!merged) {
1766 Error("MergeOutputs", "Terminate() will NOT be executed");
1767 return kFALSE;
1768 }
c57f56b7 1769 }
0f389141 1770 return kTRUE;
c57f56b7 1771}
1772
bb885a9e 1773//______________________________________________________________________________
1774void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1775{
1776// Use the output files connected to output containers from the analysis manager
1777// rather than the files defined by SetOutputFiles
1778 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1779 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1780 analysis manager");
1781 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1782}
1783
5513444a 1784//______________________________________________________________________________
1785Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1786{
1787// Start remote grid analysis.
1788
43da816a 1789 // Check if output files have to be taken from the analysis manager
1790 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1791 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1792 if (!mgr || !mgr->IsInitialized()) {
1793 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1794 return kFALSE;
43da816a 1795 }
1796 fOutputFiles = "";
1797 TIter next(mgr->GetOutputs());
1798 AliAnalysisDataContainer *output;
1799 while ((output=(AliAnalysisDataContainer*)next())) {
1800 const char *filename = output->GetFileName();
1801 if (!(strcmp(filename, "default"))) {
1802 if (!mgr->GetOutputEventHandler()) continue;
1803 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1804 }
84fcd93f 1805 if (fOutputFiles.Contains(filename)) continue;
43da816a 1806 if (fOutputFiles.Length()) fOutputFiles += " ";
1807 fOutputFiles += filename;
1808 }
c07b9ce2 1809 // Add extra files registered to the analysis manager
1810 if (mgr->GetExtraFiles().Length()) {
1811 if (fOutputFiles.Length()) fOutputFiles += " ";
1812 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1813 }
43da816a 1814 }
f7b1cbc2 1815// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1816 if (TestBit(AliAnalysisGrid::kOffline)) {
1817 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1818 \n there nor any job run. You can revise the JDL and analysis \
1819 \n macro then run the same in \"submit\" mode.");
1820 } else if (TestBit(AliAnalysisGrid::kTest)) {
1821 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1822 \n dataset.");
1823 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1824 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1825 \n space and job submitted.");
1826 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1827 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 1828 if (fMergeViaJDL) CheckInputData();
5513444a 1829 return kTRUE;
c57f56b7 1830 } else {
1831 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1832 }
1833
348be253 1834 Print();
c57f56b7 1835 if (!Connect()) {
1836 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1837 return kFALSE;
16a4353c 1838 }
348be253 1839 if (IsCheckCopy()) CheckFileCopy(gGrid->GetHomeDirectory());
c57f56b7 1840 if (!CheckInputData()) {
1841 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1842 return kFALSE;
c57f56b7 1843 }
d3339be3 1844 if (!CreateDataset(fDataPattern)) {
1845 TString serror;
1846 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
1847 if (fRunNumbers.Length()) serror = "run numbers";
1848 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
1849 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
1850 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
1851 return kFALSE;
1852 }
c57f56b7 1853 WriteAnalysisFile();
1854 WriteAnalysisMacro();
1855 WriteExecutable();
1856 WriteValidationScript();
0f389141 1857 if (fMergeViaJDL) {
1858 WriteMergingMacro();
1859 WriteMergeExecutable();
1860 WriteValidationScript(kTRUE);
1861 }
5513444a 1862 if (!CreateJDL()) return kFALSE;
1863 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1864 if (TestBit(AliAnalysisGrid::kTest)) {
1865 // Locally testing the analysis
1866 Info("StartAnalysis", "\n_______________________________________________________________________ \
1867 \n Running analysis script in a daughter shell as on a worker node \
1868 \n_______________________________________________________________________");
1869 TObjArray *list = fOutputFiles.Tokenize(" ");
1870 TIter next(list);
1871 TObjString *str;
1872 TString output_file;
1873 while((str=(TObjString*)next())) {
1874 output_file = str->GetString();
1875 Int_t index = output_file.Index("@");
1876 if (index > 0) output_file.Remove(index);
43da816a 1877 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1878 }
1879 delete list;
1880 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1881 TString validationScript = fExecutable;
1882 validationScript.ReplaceAll(".sh", "_validation.sh");
1883 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1884// gSystem->Exec("cat stdout");
5513444a 1885 return kFALSE;
c57f56b7 1886 }
5513444a 1887 // Check if submitting is managed by LPM manager
a3e84053 1888 if (fProductionMode) {
5513444a 1889 TString prodfile = fJDLName;
1890 prodfile.ReplaceAll(".jdl", ".prod");
1891 WriteProductionFile(prodfile);
1892 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1893 return kFALSE;
1894 }
a8739e8a 1895 // Submit AliEn job(s)
d2a409b2 1896 gGrid->Cd(fGridOutputDir);
a8739e8a 1897 TGridResult *res;
c57f56b7 1898 TString jobID = "";
d2a409b2 1899 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1900 // Submit a given xml or a set of runs
a8739e8a 1901 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1902 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1903 if (res) {
1904 const char *cjobId = res->GetKey(0,"jobId");
1905 if (!cjobId) {
a03be957 1906 gGrid->Stdout();
1907 gGrid->Stderr();
a8739e8a 1908 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1909 return kFALSE;
a8739e8a 1910 } else {
1911 Info("StartAnalysis", "\n_______________________________________________________________________ \
1912 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1913 \n_______________________________________________________________________",
1914 fJDLName.Data(), cjobId);
1915 jobID = cjobId;
1916 }
1917 delete res;
a03be957 1918 } else {
1919 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
1920 return kFALSE;
a8739e8a 1921 }
1922 } else {
d2a409b2 1923 // Submit for a range of enumeration of runs.
a03be957 1924 if (!Submit()) return kFALSE;
c57f56b7 1925 }
a8739e8a 1926
c57f56b7 1927 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1928 \n You may exit at any time and terminate the job later using the option <terminate> \
1929 \n ##################################################################################", jobID.Data());
bb885a9e 1930 gSystem->Exec("aliensh");
5513444a 1931 return kTRUE;
c57f56b7 1932}
1933
d2a409b2 1934//______________________________________________________________________________
a03be957 1935Bool_t AliAnalysisAlien::Submit()
d2a409b2 1936{
1937// Submit all master jobs.
1938 Int_t nmasterjobs = fInputFiles->GetEntries();
1939 Long_t tshoot = gSystem->Now();
a03be957 1940 if (!fNsubmitted && !SubmitNext()) return kFALSE;
d2a409b2 1941 while (fNsubmitted < nmasterjobs) {
1942 Long_t now = gSystem->Now();
1943 if ((now-tshoot)>30000) {
1944 tshoot = now;
a03be957 1945 if (!SubmitNext()) return kFALSE;
d2a409b2 1946 }
1947 }
a03be957 1948 return kTRUE;
d2a409b2 1949}
1950
0f389141 1951//______________________________________________________________________________
a03be957 1952Bool_t AliAnalysisAlien::SubmitMerging()
0f389141 1953{
1954// Submit all merging jobs.
1955 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1956 gGrid->Cd(fGridOutputDir);
1957 TString mergeJDLName = fExecutable;
1958 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1959 Int_t ntosubmit = fInputFiles->GetEntries();
1960 printf("### Submitting %d merging jobs...\n", ntosubmit);
1961 for (Int_t i=0; i<ntosubmit; i++) {
1962 TString query;
f866cba5 1963 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
1964 runOutDir.ReplaceAll(".xml", "");
1965 if (fOutputToRunNo)
1966 query = Form("submit %s %s", mergeJDLName.Data(), runOutDir.Data());
1967 else
1968 query = Form("submit %s %03d", mergeJDLName.Data(), i);
0f389141 1969 printf("********* %s\n",query.Data());
1970 TGridResult *res = gGrid->Command(query);
1971 if (res) {
1972 const char *cjobId = res->GetKey(0,"jobId");
1973 if (!cjobId) {
a03be957 1974 gGrid->Stdout();
1975 gGrid->Stderr();
0f389141 1976 Error("StartAnalysis", "Your JDL %s could not be submitted", mergeJDLName.Data());
a03be957 1977 return kFALSE;
0f389141 1978 } else {
1979 Info("StartAnalysis", "\n_______________________________________________________________________ \
1980 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1981 \n_______________________________________________________________________",
1982 mergeJDLName.Data(), cjobId);
1983 }
1984 delete res;
a03be957 1985 } else {
1986 Error("SubmitMerging", "No grid result after submission !!! Bailing out...");
1987 return kFALSE;
0f389141 1988 }
1989 }
a03be957 1990 if (!ntosubmit) return kTRUE;
0f389141 1991 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
1992 \n You may exit at any time and terminate the job later using the option <terminate> but disabling SetMergeViaJDL\
1993 \n ##################################################################################");
1994 gSystem->Exec("aliensh");
a03be957 1995 return kTRUE;
0f389141 1996}
1997
d2a409b2 1998//______________________________________________________________________________
a03be957 1999Bool_t AliAnalysisAlien::SubmitNext()
d2a409b2 2000{
2001// Submit next bunch of master jobs if the queue is free.
2002 static Bool_t iscalled = kFALSE;
2003 static Int_t firstmaster = 0;
2004 static Int_t lastmaster = 0;
2005 static Int_t npermaster = 0;
a03be957 2006 if (iscalled) return kTRUE;
d2a409b2 2007 iscalled = kTRUE;
2008 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2009 Int_t ntosubmit = 0;
2010 TGridResult *res;
2011 TString jobID = "";
2012 if (!fNsubmitted) ntosubmit = 1;
2013 else {
2014 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 2015 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 2016 // If last master not split, just return
a03be957 2017 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
d2a409b2 2018 // No more than 100 waiting jobs
a03be957 2019 if (nwaiting>100) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2020 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
2021 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
7586eedc 2022 if (!ntosubmit) ntosubmit = 1;
84fcd93f 2023 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 2024 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2025 }
2026 Int_t nmasterjobs = fInputFiles->GetEntries();
2027 for (Int_t i=0; i<ntosubmit; i++) {
2028 // Submit for a range of enumeration of runs.
a03be957 2029 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
d2a409b2 2030 TString query;
cd11251e 2031 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2032 runOutDir.ReplaceAll(".xml", "");
2033 if (fOutputToRunNo)
2034 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2035 else
2036 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 2037 printf("********* %s\n",query.Data());
d2a409b2 2038 res = gGrid->Command(query);
2039 if (res) {
98ca124f 2040 TString cjobId1 = res->GetKey(0,"jobId");
2041 if (!cjobId1.Length()) {
d2a409b2 2042 iscalled = kFALSE;
a03be957 2043 gGrid->Stdout();
2044 gGrid->Stderr();
2045 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2046 return kFALSE;
d2a409b2 2047 } else {
2048 Info("StartAnalysis", "\n_______________________________________________________________________ \
2049 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2050 \n_______________________________________________________________________",
98ca124f 2051 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 2052 jobID += cjobId1;
2053 jobID += " ";
98ca124f 2054 lastmaster = cjobId1.Atoi();
d2a409b2 2055 if (!firstmaster) firstmaster = lastmaster;
2056 fNsubmitted++;
2057 }
2058 delete res;
a03be957 2059 } else {
2060 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2061 return kFALSE;
d2a409b2 2062 }
2063 }
2064 iscalled = kFALSE;
a03be957 2065 return kTRUE;
d2a409b2 2066}
2067
c57f56b7 2068//______________________________________________________________________________
2069void AliAnalysisAlien::WriteAnalysisFile()
2070{
f10e8481 2071// Write current analysis manager into the file <analysisFile>
2072 TString analysisFile = fExecutable;
2073 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 2074 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2075 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2076 if (!mgr || !mgr->IsInitialized()) {
2077 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2078 return;
2079 }
2080 // Check analysis type
2081 TObject *handler;
2082 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2083 handler = (TObject*)mgr->GetInputEventHandler();
2084 if (handler) {
2085 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2086 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2087 }
2088 TDirectory *cdir = gDirectory;
f10e8481 2089 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 2090 if (file) {
fe2d7fc2 2091 // Skip task Terminate calls for the grid job
2092 mgr->SetSkipTerminate(kTRUE);
2093 // Unless merging makes no sense
2094 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 2095 mgr->Write();
2096 delete file;
fe2d7fc2 2097 // Enable termination for local jobs
2098 mgr->SetSkipTerminate(kFALSE);
c57f56b7 2099 }
2100 if (cdir) cdir->cd();
f10e8481 2101 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 2102 }
2103 Bool_t copy = kTRUE;
2104 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2105 if (copy) {
2106 CdWork();
2107 TString workdir = gGrid->GetHomeDirectory();
2108 workdir += fGridWorkingDir;
f10e8481 2109 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
2110 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
2111 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 2112 }
2113}
2114
2115//______________________________________________________________________________
2116void AliAnalysisAlien::WriteAnalysisMacro()
2117{
2118// Write the analysis macro that will steer the analysis in grid mode.
2119 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2120 ofstream out;
2121 out.open(fAnalysisMacro.Data(), ios::out);
2122 if (!out.good()) {
2123 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2124 return;
2125 }
5b273635 2126 Bool_t hasSTEERBase = kFALSE;
2127 Bool_t hasESD = kFALSE;
2128 Bool_t hasAOD = kFALSE;
2129 Bool_t hasANALYSIS = kFALSE;
2130 Bool_t hasANALYSISalice = kFALSE;
2131 Bool_t hasCORRFW = kFALSE;
c57f56b7 2132 TString func = fAnalysisMacro;
2133 TString type = "ESD";
2134 TString comment = "// Analysis using ";
2135 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
2136 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
2137 type = "AOD";
2138 comment += "AOD";
2139 }
0df6ccf2 2140 if (type!="AOD" && fFriendChainName!="") {
2141 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
2142 return;
2143 }
c57f56b7 2144 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
2145 else comment += " data";
2146 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
2147 func.ReplaceAll(".C", "");
2148 out << "void " << func.Data() << "()" << endl;
2149 out << "{" << endl;
2150 out << comment.Data() << endl;
2151 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 2152 out << " TStopwatch timer;" << endl;
2153 out << " timer.Start();" << endl << endl;
c57f56b7 2154 out << "// load base root libraries" << endl;
2155 out << " gSystem->Load(\"libTree\");" << endl;
2156 out << " gSystem->Load(\"libGeom\");" << endl;
2157 out << " gSystem->Load(\"libVMC\");" << endl;
2158 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 2159 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 2160 if (fAdditionalRootLibs.Length()) {
47a4137d 2161 // in principle libtree /lib geom libvmc etc. can go into this list, too
2162 out << "// Add aditional libraries" << endl;
2163 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2164 TIter next(list);
2165 TObjString *str;
2166 while((str=(TObjString*)next())) {
2167 if (str->GetString().Contains(".so"))
2168 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 2169 }
47a4137d 2170 if (list) delete list;
d5c6455a 2171 }
47a4137d 2172 out << "// include path" << endl;
2173 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2174 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
57377eb5 2175 out << "// Load analysis framework libraries" << endl;
4e5c5506 2176 if (!fPackages) {
4e5c5506 2177 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2178 out << " gSystem->Load(\"libESD\");" << endl;
2179 out << " gSystem->Load(\"libAOD\");" << endl;
2180 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 2181 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2182 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 2183 } else {
4e5c5506 2184 TIter next(fPackages);
2185 TObject *obj;
57377eb5 2186 TString pkgname;
5b273635 2187 TString setupPar = "AliAnalysisAlien::SetupPar";
57377eb5 2188 while ((obj=next())) {
2189 pkgname = obj->GetName();
4478e6f1 2190 if (pkgname == "STEERBase" ||
2191 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2192 if (pkgname == "ESD" ||
2193 pkgname == "ESD.par") hasESD = kTRUE;
2194 if (pkgname == "AOD" ||
2195 pkgname == "AOD.par") hasAOD = kTRUE;
2196 if (pkgname == "ANALYSIS" ||
2197 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2198 if (pkgname == "ANALYSISalice" ||
2199 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2200 if (pkgname == "CORRFW" ||
2201 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 2202 }
2203 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 2204 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2205 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 2206 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2207 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 2208 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2209 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 2210 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2211 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 2212 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2213 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 2214 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2215 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 2216 out << "// Compile other par packages" << endl;
2217 next.Reset();
fcc9bb6f 2218 while ((obj=next())) {
2219 pkgname = obj->GetName();
4478e6f1 2220 if (pkgname == "STEERBase" ||
2221 pkgname == "STEERBase.par" ||
2222 pkgname == "ESD" ||
2223 pkgname == "ESD.par" ||
2224 pkgname == "AOD" ||
2225 pkgname == "AOD.par" ||
2226 pkgname == "ANALYSIS" ||
2227 pkgname == "ANALYSIS.par" ||
2228 pkgname == "ANALYSISalice" ||
2229 pkgname == "ANALYSISalice.par" ||
2230 pkgname == "CORRFW" ||
2231 pkgname == "CORRFW.par") continue;
5b273635 2232 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 2233 }
4e5c5506 2234 }
6da75e0b 2235 if (fAdditionalLibs.Length()) {
2236 out << "// Add aditional AliRoot libraries" << endl;
2237 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2238 TIter next(list);
2239 TObjString *str;
2240 while((str=(TObjString*)next())) {
2241 if (str->GetString().Contains(".so"))
2242 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2243 }
2244 if (list) delete list;
2245 }
2246 out << endl;
c57f56b7 2247 out << "// analysis source to be compiled at runtime (if any)" << endl;
2248 if (fAnalysisSource.Length()) {
2249 TObjArray *list = fAnalysisSource.Tokenize(" ");
2250 TIter next(list);
2251 TObjString *str;
2252 while((str=(TObjString*)next())) {
2253 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2254 }
2255 if (list) delete list;
2256 }
2257 out << endl;
0f389141 2258 if (fFastReadOption) {
d3339be3 2259 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
2260 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
0f389141 2261 out << "// fast xrootd reading enabled" << endl;
2262 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2263 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
2264 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 2265 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 2266 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 2267 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2268 }
c57f56b7 2269 out << "// connect to AliEn and make the chain" << endl;
2270 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2271 if (IsUsingTags()) {
2272 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
2273 } else {
76535c98 2274 if(fFriendChainName!="AliAOD.VertexingHF.root") {
2275 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
2276 } else {
2277 out << " // Check if the macro to create the chain was provided" << endl;
2278 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
2279 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
2280 out << " return;" << endl;
2281 out << " }" << endl;
2282 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
2283 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
2284 }
c57f56b7 2285 }
2286 out << "// read the analysis manager from file" << endl;
f10e8481 2287 TString analysisFile = fExecutable;
2288 analysisFile.ReplaceAll(".sh", ".root");
2289 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 2290 out << " if (!file) return;" << endl;
2291 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2292 out << " AliAnalysisManager *mgr = 0;" << endl;
2293 out << " TKey *key;" << endl;
2294 out << " while ((key=(TKey*)nextkey())) {" << endl;
2295 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2296 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2297 out << " };" << endl;
2298 out << " if (!mgr) {" << endl;
f10e8481 2299 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 2300 out << " return;" << endl;
2301 out << " }" << endl << endl;
2302 out << " mgr->PrintStatus();" << endl;
52b6a92b 2303 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 2304 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 2305 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 2306 } else {
2307 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 2308 }
2309 }
c57f56b7 2310 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 2311 out << " timer.Stop();" << endl;
2312 out << " timer.Print();" << endl;
c57f56b7 2313 out << "}" << endl << endl;
2314 if (IsUsingTags()) {
2315 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
2316 out << "{" << endl;
2317 out << "// Create a chain using tags from the xml file." << endl;
2318 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
2319 out << " if (!coll) {" << endl;
2320 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2321 out << " return NULL;" << endl;
2322 out << " }" << endl;
2323 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
2324 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
2325 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
2326 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
2327 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
2328 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
2329 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
2330 out << " // Check if the cuts configuration file was provided" << endl;
2331 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
2332 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
2333 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2334 out << " }" << endl;
0df6ccf2 2335 if (fFriendChainName=="") {
2336 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2337 } else {
2338 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
2339 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
2340 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
2341 }
c57f56b7 2342 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
2343 out << " chain->ls();" << endl;
2344 out << " return chain;" << endl;
fcc9bb6f 2345 out << "}" << endl << endl;
c57f56b7 2346 if (gSystem->AccessPathName("ConfigureCuts.C")) {
2347 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
2348 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
2349 msg += " AliLHCTagCuts *lhcCuts,\n";
2350 msg += " AliDetectorTagCuts *detCuts,\n";
2351 msg += " AliEventTagCuts *evCuts)";
2352 Info("WriteAnalysisMacro", msg.Data());
2353 }
0df6ccf2 2354 }
2355 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 2356 out <<"//________________________________________________________________________________" << endl;
c57f56b7 2357 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
2358 out << "{" << endl;
2359 out << "// Create a chain using url's from xml file" << endl;
2360 out << " TString treename = type;" << endl;
2361 out << " treename.ToLower();" << endl;
2362 out << " treename += \"Tree\";" << endl;
e02fee64 2363 out << " printf(\"***************************************\\n\");" << endl;
2364 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
2365 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 2366 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
2367 out << " if (!coll) {" << endl;
2368 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2369 out << " return NULL;" << endl;
2370 out << " }" << endl;
2371 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 2372 if(fFriendChainName!="") {
2373 out << " TChain *chainFriend = new TChain(treename);" << endl;
2374 }
c57f56b7 2375 out << " coll->Reset();" << endl;
0df6ccf2 2376 out << " while (coll->Next()) {" << endl;
2377 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
2378 if(fFriendChainName!="") {
2379 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
2380 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2381 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2382 out << " chainFriend->Add(fileFriend.Data());" << endl;
2383 }
2384 out << " }" << endl;
c57f56b7 2385 out << " if (!chain->GetNtrees()) {" << endl;
2386 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
2387 out << " return NULL;" << endl;
2388 out << " }" << endl;
0df6ccf2 2389 if(fFriendChainName!="") {
2390 out << " chain->AddFriend(chainFriend);" << endl;
2391 }
c57f56b7 2392 out << " return chain;" << endl;
fcc9bb6f 2393 out << "}" << endl << endl;
c57f56b7 2394 }
5b273635 2395 if (hasANALYSISalice) {
2396 out <<"//________________________________________________________________________________" << endl;
2397 out << "Bool_t SetupPar(const char *package) {" << endl;
2398 out << "// Compile the package and set it up." << endl;
2399 out << " TString pkgdir = package;" << endl;
2400 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2401 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2402 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2403 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2404 out << " // Check for BUILD.sh and execute" << endl;
2405 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2406 out << " printf(\"*******************************\\n\");" << endl;
2407 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2408 out << " printf(\"*******************************\\n\");" << endl;
2409 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2410 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2411 out << " gSystem->ChangeDirectory(cdir);" << endl;
2412 out << " return kFALSE;" << endl;
2413 out << " }" << endl;
2414 out << " } else {" << endl;
2415 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2416 out << " gSystem->ChangeDirectory(cdir);" << endl;
2417 out << " return kFALSE;" << endl;
2418 out << " }" << endl;
2419 out << " // Check for SETUP.C and execute" << endl;
2420 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2421 out << " printf(\"*******************************\\n\");" << endl;
2422 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2423 out << " printf(\"*******************************\\n\");" << endl;
2424 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2425 out << " } else {" << endl;
2426 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2427 out << " gSystem->ChangeDirectory(cdir);" << endl;
2428 out << " return kFALSE;" << endl;
2429 out << " }" << endl;
2430 out << " // Restore original workdir" << endl;
2431 out << " gSystem->ChangeDirectory(cdir);" << endl;
2432 out << " return kTRUE;" << endl;
2433 out << "}" << endl;
2434 }
c57f56b7 2435 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2436 }
2437 Bool_t copy = kTRUE;
2438 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2439 if (copy) {
2440 CdWork();
2441 TString workdir = gGrid->GetHomeDirectory();
2442 workdir += fGridWorkingDir;
2443 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2444 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2445 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2446 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2447 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2448 }
2449 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2450 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2451 }
2452}
2453
0f389141 2454//______________________________________________________________________________
2455void AliAnalysisAlien::WriteMergingMacro()
2456{
2457// Write a macro to merge the outputs per master job.
2458 if (!fMergeViaJDL) return;
2459 if (!fOutputFiles.Length()) {
2460 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2461 return;
2462 }
2463 TString mergingMacro = fExecutable;
2464 mergingMacro.ReplaceAll(".sh","_merge.C");
2465 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2466 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2467 ofstream out;
2468 out.open(mergingMacro.Data(), ios::out);
2469 if (!out.good()) {
2470 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2471 return;
2472 }
5b273635 2473 Bool_t hasSTEERBase = kFALSE;
2474 Bool_t hasESD = kFALSE;
2475 Bool_t hasAOD = kFALSE;
2476 Bool_t hasANALYSIS = kFALSE;
2477 Bool_t hasANALYSISalice = kFALSE;
2478 Bool_t hasCORRFW = kFALSE;
0f389141 2479 TString func = mergingMacro;
2480 TString comment;
2481 func.ReplaceAll(".C", "");
2482 out << "void " << func.Data() << "(const char *dir)" << endl;
2483 out << "{" << endl;
2484 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
2485 out << " TStopwatch timer;" << endl;
2486 out << " timer.Start();" << endl << endl;
2487 out << "// load base root libraries" << endl;
2488 out << " gSystem->Load(\"libTree\");" << endl;
2489 out << " gSystem->Load(\"libGeom\");" << endl;
2490 out << " gSystem->Load(\"libVMC\");" << endl;
2491 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
2492 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
2493 if (fAdditionalRootLibs.Length()) {
2494 // in principle libtree /lib geom libvmc etc. can go into this list, too
2495 out << "// Add aditional libraries" << endl;
2496 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2497 TIter next(list);
2498 TObjString *str;
2499 while((str=(TObjString*)next())) {
2500 if (str->GetString().Contains(".so"))
2501 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2502 }
2503 if (list) delete list;
2504 }
2505 out << "// include path" << endl;
2506 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2507 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
2508 out << "// Load analysis framework libraries" << endl;
2509 if (!fPackages) {
2510 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2511 out << " gSystem->Load(\"libESD\");" << endl;
2512 out << " gSystem->Load(\"libAOD\");" << endl;
2513 out << " gSystem->Load(\"libANALYSIS\");" << endl;
2514 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2515 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
2516 } else {
2517 TIter next(fPackages);
2518 TObject *obj;
2519 TString pkgname;
5b273635 2520 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 2521 while ((obj=next())) {
2522 pkgname = obj->GetName();
2523 if (pkgname == "STEERBase" ||
2524 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2525 if (pkgname == "ESD" ||
2526 pkgname == "ESD.par") hasESD = kTRUE;
2527 if (pkgname == "AOD" ||
2528 pkgname == "AOD.par") hasAOD = kTRUE;
2529 if (pkgname == "ANALYSIS" ||
2530 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2531 if (pkgname == "ANALYSISalice" ||
2532 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2533 if (pkgname == "CORRFW" ||
2534 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
2535 }
5b273635 2536 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 2537 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2538 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 2539 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2540 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 2541 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2542 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 2543 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2544 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 2545 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2546 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 2547 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2548 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 2549 out << "// Compile other par packages" << endl;
2550 next.Reset();
2551 while ((obj=next())) {
2552 pkgname = obj->GetName();
2553 if (pkgname == "STEERBase" ||
2554 pkgname == "STEERBase.par" ||
2555 pkgname == "ESD" ||
2556 pkgname == "ESD.par" ||
2557 pkgname == "AOD" ||
2558 pkgname == "AOD.par" ||
2559 pkgname == "ANALYSIS" ||
2560 pkgname == "ANALYSIS.par" ||
2561 pkgname == "ANALYSISalice" ||
2562 pkgname == "ANALYSISalice.par" ||
2563 pkgname == "CORRFW" ||
2564 pkgname == "CORRFW.par") continue;
5b273635 2565 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 2566 }
2567 }
2568 if (fAdditionalLibs.Length()) {
2569 out << "// Add aditional AliRoot libraries" << endl;
2570 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2571 TIter next(list);
2572 TObjString *str;
2573 while((str=(TObjString*)next())) {
2574 if (str->GetString().Contains(".so"))
2575 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2576 }
2577 if (list) delete list;
2578 }
2579 out << endl;
2580 out << "// Analysis source to be compiled at runtime (if any)" << endl;
2581 if (fAnalysisSource.Length()) {
2582 TObjArray *list = fAnalysisSource.Tokenize(" ");
2583 TIter next(list);
2584 TObjString *str;
2585 while((str=(TObjString*)next())) {
2586 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2587 }
2588 if (list) delete list;
2589 }
2590 out << endl;
2591 if (fFastReadOption) {
2592 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
2593 out << "// fast xrootd reading enabled" << endl;
2594 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
d3339be3 2595 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",10);" << endl;
2596 out << " gEnv->SetValue(\"XNet.RequestTimeout\",10);" << endl;
0f389141 2597 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
d3339be3 2598 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",10);" << endl;
0f389141 2599 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2600 }
2601 out << "// Connect to AliEn" << endl;
2602 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2603 out << " TString outputDir = \"" << fGridOutputDir << "/\";" << endl;
2604 out << " outputDir += dir;" << endl;
2605 out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
2606 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
1f0d1ca2 2607 out << " mergeExcludes += \"" << AliAnalysisManager::GetAnalysisManager()->GetExtraFiles() << "\";" << endl;
0f389141 2608 out << " TObjArray *list = outputFiles.Tokenize(\" \");" << endl;
2609 out << " TIter *iter = new TIter(list);" << endl;
2610 out << " TObjString *str;" << endl;
2611 out << " TString output_file;" << endl;
2612 out << " Bool_t merged = kTRUE;" << endl;
2613 out << " while((str=(TObjString*)iter->Next())) {" << endl;
2614 out << " output_file = str->GetString();" << endl;
2615 out << " Int_t index = output_file.Index(\"@\");" << endl;
2616 out << " if (index > 0) output_file.Remove(index);" << endl;
2617 out << " // Skip already merged outputs" << endl;
2618 out << " if (!gSystem->AccessPathName(output_file)) {" << endl;
2619 out << " printf(\"Output file <%s> found. Not merging again.\",output_file.Data());" << endl;
2620 out << " continue;" << endl;
2621 out << " }" << endl;
2622 out << " if (mergeExcludes.Contains(output_file.Data())) continue;" << endl;
2623 out << " merged = AliAnalysisAlien::MergeOutput(output_file, outputDir, " << fMaxMergeFiles << ");" << endl;
2624 out << " if (!merged) {" << endl;
2625 out << " printf(\"ERROR: Cannot merge %s\\n\", output_file.Data());" << endl;
0f389141 2626 out << " }" << endl;
2627 out << " }" << endl;
f866cba5 2628 out << "// read the analysis manager from file" << endl;
2629 TString analysisFile = fExecutable;
2630 analysisFile.ReplaceAll(".sh", ".root");
2631 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
2632 out << " if (!file) return;" << endl;
2633 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2634 out << " AliAnalysisManager *mgr = 0;" << endl;
2635 out << " TKey *key;" << endl;
2636 out << " while ((key=(TKey*)nextkey())) {" << endl;
2637 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2638 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2639 out << " };" << endl;
2640 out << " if (!mgr) {" << endl;
2641 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
2642 out << " return;" << endl;
2643 out << " }" << endl << endl;
b385fec0 2644 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 2645 out << " mgr->PrintStatus();" << endl;
2646 if (AliAnalysisManager::GetAnalysisManager()) {
2647 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
2648 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
2649 } else {
2650 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
2651 }
2652 }
2653 out << " mgr->StartAnalysis(\"gridterminate\");" << endl;
0f389141 2654 out << "}" << endl << endl;
5b273635 2655 if (hasANALYSISalice) {
2656 out <<"//________________________________________________________________________________" << endl;
2657 out << "Bool_t SetupPar(const char *package) {" << endl;
2658 out << "// Compile the package and set it up." << endl;
2659 out << " TString pkgdir = package;" << endl;
2660 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2661 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2662 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2663 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2664 out << " // Check for BUILD.sh and execute" << endl;
2665 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2666 out << " printf(\"*******************************\\n\");" << endl;
2667 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2668 out << " printf(\"*******************************\\n\");" << endl;
2669 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2670 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2671 out << " gSystem->ChangeDirectory(cdir);" << endl;
2672 out << " return kFALSE;" << endl;
2673 out << " }" << endl;
2674 out << " } else {" << endl;
2675 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2676 out << " gSystem->ChangeDirectory(cdir);" << endl;
2677 out << " return kFALSE;" << endl;
2678 out << " }" << endl;
2679 out << " // Check for SETUP.C and execute" << endl;
2680 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2681 out << " printf(\"*******************************\\n\");" << endl;
2682 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2683 out << " printf(\"*******************************\\n\");" << endl;
2684 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2685 out << " } else {" << endl;
2686 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2687 out << " gSystem->ChangeDirectory(cdir);" << endl;
2688 out << " return kFALSE;" << endl;
2689 out << " }" << endl;
2690 out << " // Restore original workdir" << endl;
2691 out << " gSystem->ChangeDirectory(cdir);" << endl;
2692 out << " return kTRUE;" << endl;
2693 out << "}" << endl;
2694 }
0f389141 2695 }
2696 Bool_t copy = kTRUE;
2697 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2698 if (copy) {
2699 CdWork();
2700 TString workdir = gGrid->GetHomeDirectory();
2701 workdir += fGridWorkingDir;
2702 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
2703 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
2704 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
2705 }
2706}
2707
2708//______________________________________________________________________________
2709Bool_t AliAnalysisAlien::SetupPar(const char *package)
2710{
2711// Compile the par file archive pointed by <package>. This must be present in the current durectory.
2712// Note that for loading the compiled library. The current directory should have precedence in
2713// LD_LIBRARY_PATH
2714 TString pkgdir = package;
2715 pkgdir.ReplaceAll(".par","");
2716 gSystem->Exec(Form("tar xvzf %s.par", pkgdir.Data()));
2717 TString cdir = gSystem->WorkingDirectory();
2718 gSystem->ChangeDirectory(pkgdir);
2719 // Check for BUILD.sh and execute
2720 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
2721 printf("**************************************************\n");
2722 printf("*** Building PAR archive %s\n", package);
2723 printf("**************************************************\n");
2724 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
2725 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
2726 gSystem->ChangeDirectory(cdir);
2727 return kFALSE;
2728 }
2729 } else {
2730 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
2731 gSystem->ChangeDirectory(cdir);
2732 return kFALSE;
2733 }
2734 // Check for SETUP.C and execute
2735 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
2736 printf("**************************************************\n");
2737 printf("*** Setup PAR archive %s\n", package);
2738 printf("**************************************************\n");
2739 gROOT->Macro("PROOF-INF/SETUP.C");
2740 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
2741 } else {
2742 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
2743 gSystem->ChangeDirectory(cdir);
2744 return kFALSE;
2745 }
2746 // Restore original workdir
2747 gSystem->ChangeDirectory(cdir);
2748 return kTRUE;
2749}
2750
c57f56b7 2751//______________________________________________________________________________
2752void AliAnalysisAlien::WriteExecutable()
2753{
2754// Generate the alien executable script.
2755 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2756 ofstream out;
2757 out.open(fExecutable.Data(), ios::out);
2758 if (out.bad()) {
5513444a 2759 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2760 return;
2761 }
2762 out << "#!/bin/bash" << endl;
c57f56b7 2763 out << "echo \"=========================================\"" << endl;
2764 out << "echo \"############## PATH : ##############\"" << endl;
2765 out << "echo $PATH" << endl;
2766 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2767 out << "echo $LD_LIBRARY_PATH" << endl;
2768 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2769 out << "echo $ROOTSYS" << endl;
2770 out << "echo \"############## which root : ##############\"" << endl;
2771 out << "which root" << endl;
2772 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2773 out << "echo $ALICE_ROOT" << endl;
2774 out << "echo \"############## which aliroot : ##############\"" << endl;
2775 out << "which aliroot" << endl;
9c5ddadc 2776 out << "echo \"############## system limits : ##############\"" << endl;
2777 out << "ulimit -a" << endl;
2778 out << "echo \"############## memory : ##############\"" << endl;
2779 out << "free -m" << endl;
c57f56b7 2780 out << "echo \"=========================================\"" << endl << endl;
74a53467 2781 // Make sure we can properly compile par files
2782 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2783 out << fExecutableCommand << " ";
631c0b05 2784 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2785 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2786 out << "echo \"############## memory after: ##############\"" << endl;
2787 out << "free -m" << endl;
2788 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2789 out << "dmesg | tail -n 10" << endl;
c57f56b7 2790 }
2791 Bool_t copy = kTRUE;
2792 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2793 if (copy) {
2794 CdWork();
2795 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2796 TString bindir = Form("%s/bin", workdir.Data());
2797 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2798 workdir += fGridWorkingDir;
2799 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2800 if (FileExists(executable)) gGrid->Rm(executable);
2801 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2802 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2803 }
2804}
2805
0f389141 2806//______________________________________________________________________________
2807void AliAnalysisAlien::WriteMergeExecutable()
2808{
2809// Generate the alien executable script for the merging job.
2810 if (!fMergeViaJDL) return;
2811 TString mergeExec = fExecutable;
2812 mergeExec.ReplaceAll(".sh", "_merge.sh");
2813 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2814 ofstream out;
2815 out.open(mergeExec.Data(), ios::out);
2816 if (out.bad()) {
2817 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
2818 return;
2819 }
2820 out << "#!/bin/bash" << endl;
2821 out << "echo \"=========================================\"" << endl;
2822 out << "echo \"############## PATH : ##############\"" << endl;
2823 out << "echo $PATH" << endl;
2824 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2825 out << "echo $LD_LIBRARY_PATH" << endl;
2826 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2827 out << "echo $ROOTSYS" << endl;
2828 out << "echo \"############## which root : ##############\"" << endl;
2829 out << "which root" << endl;
2830 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2831 out << "echo $ALICE_ROOT" << endl;
2832 out << "echo \"############## which aliroot : ##############\"" << endl;
2833 out << "which aliroot" << endl;
2834 out << "echo \"############## system limits : ##############\"" << endl;
2835 out << "ulimit -a" << endl;
2836 out << "echo \"############## memory : ##############\"" << endl;
2837 out << "free -m" << endl;
2838 out << "echo \"=========================================\"" << endl << endl;
2839 // Make sure we can properly compile par files
2840 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
2841 TString mergeMacro = fExecutable;
2842 mergeMacro.ReplaceAll(".sh", "_merge.C");
2843 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
2844 out << fExecutableCommand << " " << "$ARG" << endl;
2845 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
2846 out << "echo \"############## memory after: ##############\"" << endl;
2847 out << "free -m" << endl;
2848 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2849 out << "dmesg | tail -n 10" << endl;
2850 }
2851 Bool_t copy = kTRUE;
2852 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2853 if (copy) {
2854 CdWork();
2855 TString workdir = gGrid->GetHomeDirectory();
2856 TString bindir = Form("%s/bin", workdir.Data());
2857 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
2858 workdir += fGridWorkingDir;
2859 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
2860 if (FileExists(executable)) gGrid->Rm(executable);
2861 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
2862 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
2863 }
2864}
2865
c57f56b7 2866//______________________________________________________________________________
5513444a 2867void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2868{
2869// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2870// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2871// Next lines: full_path_to_dataset XXX (XXX is a string)
2872// To submit, one has to: submit jdl XXX for all lines
2873 ofstream out;
2874 out.open(filename, ios::out);
2875 if (out.bad()) {
2876 Error("WriteProductionFile", "Bad file name: %s", filename);
2877 return;
2878 }
2879 TString workdir = gGrid->GetHomeDirectory();
2880 workdir += fGridWorkingDir;
f5e8c702 2881 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2882 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2883 out << locjdl << " " << njobspermaster << endl;
5513444a 2884 Int_t nmasterjobs = fInputFiles->GetEntries();
2885 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 2886 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2887 runOutDir.ReplaceAll(".xml", "");
2888 if (fOutputToRunNo)
2889 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
2890 else
2891 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2892 }
2893 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
fdbbc7be 2894 if (FileExists(filename)) gGrid->Rm(filename);
5513444a 2895 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2896}
2897
2898//______________________________________________________________________________
0f389141 2899void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 2900{
2901// Generate the alien validation script.
2902 // Generate the validation script
2903 TObjString *os;
0d5d317c 2904 TString validationScript = fExecutable;
0f389141 2905 if (merge) validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
2906 else validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2907 if (!Connect()) {
2908 Error("WriteValidationScript", "Alien connection required");
2909 return;
2910 }
2911 TString out_stream = "";
2912 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2913 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2914 ofstream out;
0d5d317c 2915 out.open(validationScript, ios::out);
c57f56b7 2916 out << "#!/bin/bash" << endl;
2917 out << "##################################################" << endl;
2918 out << "validateout=`dirname $0`" << endl;
2919 out << "validatetime=`date`" << endl;
2920 out << "validated=\"0\";" << endl;
2921 out << "error=0" << endl;
2922 out << "if [ -z $validateout ]" << endl;
2923 out << "then" << endl;
2924 out << " validateout=\".\"" << endl;
2925 out << "fi" << endl << endl;
2926 out << "cd $validateout;" << endl;
2927 out << "validateworkdir=`pwd`;" << endl << endl;
2928 out << "echo \"*******************************************************\"" << out_stream << endl;
2929 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2930 out << "" << endl;
2931 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2932 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2933 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2934 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2935 out << "ls -la ./" << out_stream << endl;
2936 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2937 out << "##################################################" << endl;
ebec370a 2938
2939 out << "" << endl;
2940 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2941 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2942 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2943 out << "" << endl;
2944
2945 out << "if [ ! -f stderr ] ; then" << endl;
2946 out << " error=1" << endl;
2947 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2948 out << " echo \"Error = $error\" " << out_stream << endl;
2949 out << "fi" << endl;
2950
2951 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2952 out << " error=1" << endl;
2953 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2954 out << " echo \"$parArch\" " << out_stream << endl;
2955 out << " echo \"Error = $error\" " << out_stream << endl;
2956 out << "fi" << endl;
2957
2958 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2959 out << " error=1" << endl;
2960 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2961 out << " echo \"$segViol\" " << out_stream << endl;
2962 out << " echo \"Error = $error\" " << out_stream << endl;
2963 out << "fi" << endl;
2964
2965 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2966 out << " error=1" << endl;
2967 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2968 out << " echo \"$segFault\" " << out_stream << endl;
2969 out << " echo \"Error = $error\" " << out_stream << endl;
2970 out << "fi" << endl;
2971
2972 // Part dedicated to the specific analyses running into the train
2973
c57f56b7 2974 TObjArray *arr = fOutputFiles.Tokenize(" ");
2975 TIter next1(arr);
2976 TString output_file;
5b9b4998 2977 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2978 TString extra = mgr->GetExtraFiles();
c57f56b7 2979 while ((os=(TObjString*)next1())) {
2980 output_file = os->GetString();
2981 Int_t index = output_file.Index("@");
2982 if (index > 0) output_file.Remove(index);
0f389141 2983 if (merge && fMergeExcludes.Contains(output_file)) continue;
5b9b4998 2984 if (extra.Contains(output_file)) continue;
c57f56b7 2985 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2986 out << " error=1" << endl;
2987 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2988 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2989 out << "fi" << endl;
2990 }
2991 delete arr;
f866cba5 2992 if (!merge) {
2993 out << "if ! [ -f outputs_valid ] ; then" << endl;
2994 out << " error=1" << endl;
2995 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2996 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2997 out << "fi" << endl;
2998 }
923e2ca5 2999
c57f56b7 3000 out << "if [ $error = 0 ] ; then" << endl;
3001 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
3002 out << "fi" << endl;
3003
3004 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
3005 out << "echo \"*******************************************************\"" << out_stream << endl;
3006 out << "cd -" << endl;
3007 out << "exit $error" << endl;
3008 }
3009 Bool_t copy = kTRUE;
3010 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3011 if (copy) {
3012 CdWork();
3013 TString workdir = gGrid->GetHomeDirectory();
3014 workdir += fGridWorkingDir;
0d5d317c 3015 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
3016 if (FileExists(validationScript)) gGrid->Rm(validationScript);
3017 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 3018 }
3019}