]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Skip local Terminate if there was a merging+terminate job
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
0f389141 24#include "TEnv.h"
25#include "TError.h"
c57f56b7 26#include "TROOT.h"
27#include "TSystem.h"
28#include "TFile.h"
29#include "TObjString.h"
30#include "TObjArray.h"
31#include "TGrid.h"
32#include "TGridResult.h"
33#include "TGridCollection.h"
34#include "TGridJDL.h"
d2a409b2 35#include "TGridJobStatusList.h"
36#include "TGridJobStatus.h"
c57f56b7 37#include "TFileMerger.h"
38#include "AliAnalysisManager.h"
bb885a9e 39#include "AliVEventHandler.h"
40#include "AliAnalysisDataContainer.h"
c57f56b7 41#include "AliAnalysisAlien.h"
42
43ClassImp(AliAnalysisAlien)
44
45//______________________________________________________________________________
46AliAnalysisAlien::AliAnalysisAlien()
47 :AliAnalysisGrid(),
48 fGridJDL(NULL),
0f389141 49 fMergingJDL(NULL),
c57f56b7 50 fPrice(0),
51 fTTL(0),
52 fSplitMaxInputFileNumber(0),
53 fMaxInitFailed(0),
54 fMasterResubmitThreshold(0),
bb885a9e 55 fNtestFiles(0),
319593fb 56 fNrunsPerMaster(0),
16a4353c 57 fMaxMergeFiles(0),
d2a409b2 58 fNsubmitted(0),
a3e84053 59 fProductionMode(0),
cd11251e 60 fOutputToRunNo(0),
0f389141 61 fMergeViaJDL(0),
62 fFastReadOption(0),
63 fOverwriteMode(0),
c57f56b7 64 fRunNumbers(),
65 fExecutable(),
0a1c1f7f 66 fExecutableCommand(),
c57f56b7 67 fArguments(),
631c0b05 68 fExecutableArgs(),
c57f56b7 69 fAnalysisMacro(),
70 fAnalysisSource(),
d5c6455a 71 fAdditionalRootLibs(),
c57f56b7 72 fAdditionalLibs(),
73 fSplitMode(),
74 fAPIVersion(),
75 fROOTVersion(),
76 fAliROOTVersion(),
648174cf 77 fExternalPackages(),
c57f56b7 78 fUser(),
79 fGridWorkingDir(),
80 fGridDataDir(),
81 fDataPattern(),
82 fGridOutputDir(),
83 fOutputArchive(),
84 fOutputFiles(),
85 fInputFormat(),
e7c71df0 86 fDatasetName(),
c57f56b7 87 fJDLName(),
bb885a9e 88 fMergeExcludes(),
f965131e 89 fIncludePath(),
bb885a9e 90 fCloseSE(),
0df6ccf2 91 fFriendChainName(),
c6cb3634 92 fJobTag(),
648174cf 93 fOutputSingle(),
5fce53f4 94 fRunPrefix(),
4e5c5506 95 fInputFiles(0),
96 fPackages(0)
c57f56b7 97{
98// Dummy ctor.
99 SetDefaults();
100}
101
102//______________________________________________________________________________
103AliAnalysisAlien::AliAnalysisAlien(const char *name)
104 :AliAnalysisGrid(name),
105 fGridJDL(NULL),
0f389141 106 fMergingJDL(NULL),
c57f56b7 107 fPrice(0),
108 fTTL(0),
109 fSplitMaxInputFileNumber(0),
110 fMaxInitFailed(0),
111 fMasterResubmitThreshold(0),
bb885a9e 112 fNtestFiles(0),
319593fb 113 fNrunsPerMaster(0),
16a4353c 114 fMaxMergeFiles(0),
d2a409b2 115 fNsubmitted(0),
a3e84053 116 fProductionMode(0),
cd11251e 117 fOutputToRunNo(0),
0f389141 118 fMergeViaJDL(0),
119 fFastReadOption(0),
120 fOverwriteMode(0),
c57f56b7 121 fRunNumbers(),
122 fExecutable(),
0a1c1f7f 123 fExecutableCommand(),
c57f56b7 124 fArguments(),
631c0b05 125 fExecutableArgs(),
c57f56b7 126 fAnalysisMacro(),
127 fAnalysisSource(),
d5c6455a 128 fAdditionalRootLibs(),
c57f56b7 129 fAdditionalLibs(),
130 fSplitMode(),
131 fAPIVersion(),
132 fROOTVersion(),
133 fAliROOTVersion(),
648174cf 134 fExternalPackages(),
c57f56b7 135 fUser(),
136 fGridWorkingDir(),
137 fGridDataDir(),
138 fDataPattern(),
139 fGridOutputDir(),
140 fOutputArchive(),
141 fOutputFiles(),
142 fInputFormat(),
e7c71df0 143 fDatasetName(),
c57f56b7 144 fJDLName(),
bb885a9e 145 fMergeExcludes(),
f965131e 146 fIncludePath(),
bb885a9e 147 fCloseSE(),
0df6ccf2 148 fFriendChainName(),
c6cb3634 149 fJobTag(),
648174cf 150 fOutputSingle(),
5fce53f4 151 fRunPrefix(),
4e5c5506 152 fInputFiles(0),
153 fPackages(0)
c57f56b7 154{
155// Default ctor.
156 SetDefaults();
157}
158
159//______________________________________________________________________________
160AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
161 :AliAnalysisGrid(other),
162 fGridJDL(NULL),
0f389141 163 fMergingJDL(NULL),
c57f56b7 164 fPrice(other.fPrice),
165 fTTL(other.fTTL),
166 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
167 fMaxInitFailed(other.fMaxInitFailed),
168 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 169 fNtestFiles(other.fNtestFiles),
319593fb 170 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 171 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 172 fNsubmitted(other.fNsubmitted),
a3e84053 173 fProductionMode(other.fProductionMode),
cd11251e 174 fOutputToRunNo(other.fOutputToRunNo),
0f389141 175 fMergeViaJDL(other.fMergeViaJDL),
176 fFastReadOption(other.fFastReadOption),
177 fOverwriteMode(other.fOverwriteMode),
c57f56b7 178 fRunNumbers(other.fRunNumbers),
179 fExecutable(other.fExecutable),
0a1c1f7f 180 fExecutableCommand(other.fExecutableCommand),
c57f56b7 181 fArguments(other.fArguments),
631c0b05 182 fExecutableArgs(other.fExecutableArgs),
c57f56b7 183 fAnalysisMacro(other.fAnalysisMacro),
184 fAnalysisSource(other.fAnalysisSource),
d5c6455a 185 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 186 fAdditionalLibs(other.fAdditionalLibs),
187 fSplitMode(other.fSplitMode),
188 fAPIVersion(other.fAPIVersion),
189 fROOTVersion(other.fROOTVersion),
190 fAliROOTVersion(other.fAliROOTVersion),
648174cf 191 fExternalPackages(other.fExternalPackages),
c57f56b7 192 fUser(other.fUser),
193 fGridWorkingDir(other.fGridWorkingDir),
194 fGridDataDir(other.fGridDataDir),
195 fDataPattern(other.fDataPattern),
196 fGridOutputDir(other.fGridOutputDir),
197 fOutputArchive(other.fOutputArchive),
198 fOutputFiles(other.fOutputFiles),
199 fInputFormat(other.fInputFormat),
e7c71df0 200 fDatasetName(other.fDatasetName),
c57f56b7 201 fJDLName(other.fJDLName),
bb885a9e 202 fMergeExcludes(other.fMergeExcludes),
f965131e 203 fIncludePath(other.fIncludePath),
bb885a9e 204 fCloseSE(other.fCloseSE),
0df6ccf2 205 fFriendChainName(other.fFriendChainName),
c6cb3634 206 fJobTag(other.fJobTag),
648174cf 207 fOutputSingle(other.fOutputSingle),
5fce53f4 208 fRunPrefix(other.fRunPrefix),
4e5c5506 209 fInputFiles(0),
210 fPackages(0)
c57f56b7 211{
212// Copy ctor.
213 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 214 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 215 fRunRange[0] = other.fRunRange[0];
216 fRunRange[1] = other.fRunRange[1];
c57f56b7 217 if (other.fInputFiles) {
218 fInputFiles = new TObjArray();
219 TIter next(other.fInputFiles);
220 TObject *obj;
221 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
222 fInputFiles->SetOwner();
223 }
4e5c5506 224 if (other.fPackages) {
225 fPackages = new TObjArray();
226 TIter next(other.fPackages);
227 TObject *obj;
228 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
229 fPackages->SetOwner();
230 }
c57f56b7 231}
232
233//______________________________________________________________________________
234AliAnalysisAlien::~AliAnalysisAlien()
235{
236// Destructor.
237 if (fGridJDL) delete fGridJDL;
0f389141 238 if (fMergingJDL) delete fMergingJDL;
c57f56b7 239 if (fInputFiles) delete fInputFiles;
4e5c5506 240 if (fPackages) delete fPackages;
c57f56b7 241}
242
243//______________________________________________________________________________
244AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
245{
246// Assignment.
247 if (this != &other) {
248 AliAnalysisGrid::operator=(other);
249 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 250 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 251 fPrice = other.fPrice;
252 fTTL = other.fTTL;
253 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
254 fMaxInitFailed = other.fMaxInitFailed;
255 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 256 fNtestFiles = other.fNtestFiles;
a3e84053 257 fNrunsPerMaster = other.fNrunsPerMaster;
258 fMaxMergeFiles = other.fMaxMergeFiles;
259 fNsubmitted = other.fNsubmitted;
260 fProductionMode = other.fProductionMode;
cd11251e 261 fOutputToRunNo = other.fOutputToRunNo;
0f389141 262 fMergeViaJDL = other.fMergeViaJDL;
263 fFastReadOption = other.fFastReadOption;
264 fOverwriteMode = other.fOverwriteMode;
c57f56b7 265 fRunNumbers = other.fRunNumbers;
266 fExecutable = other.fExecutable;
0a1c1f7f 267 fExecutableCommand = other.fExecutableCommand;
c57f56b7 268 fArguments = other.fArguments;
631c0b05 269 fExecutableArgs = other.fExecutableArgs;
c57f56b7 270 fAnalysisMacro = other.fAnalysisMacro;
271 fAnalysisSource = other.fAnalysisSource;
d5c6455a 272 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 273 fAdditionalLibs = other.fAdditionalLibs;
274 fSplitMode = other.fSplitMode;
275 fAPIVersion = other.fAPIVersion;
276 fROOTVersion = other.fROOTVersion;
277 fAliROOTVersion = other.fAliROOTVersion;
648174cf 278 fExternalPackages = other.fExternalPackages;
c57f56b7 279 fUser = other.fUser;
280 fGridWorkingDir = other.fGridWorkingDir;
281 fGridDataDir = other.fGridDataDir;
282 fDataPattern = other.fDataPattern;
283 fGridOutputDir = other.fGridOutputDir;
284 fOutputArchive = other.fOutputArchive;
285 fOutputFiles = other.fOutputFiles;
286 fInputFormat = other.fInputFormat;
e7c71df0 287 fDatasetName = other.fDatasetName;
c57f56b7 288 fJDLName = other.fJDLName;
bb885a9e 289 fMergeExcludes = other.fMergeExcludes;
f965131e 290 fIncludePath = other.fIncludePath;
bb885a9e 291 fCloseSE = other.fCloseSE;
0df6ccf2 292 fFriendChainName = other.fFriendChainName;
c6cb3634 293 fJobTag = other.fJobTag;
648174cf 294 fOutputSingle = other.fOutputSingle;
5fce53f4 295 fRunPrefix = other.fRunPrefix;
c57f56b7 296 if (other.fInputFiles) {
297 fInputFiles = new TObjArray();
298 TIter next(other.fInputFiles);
299 TObject *obj;
300 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
301 fInputFiles->SetOwner();
302 }
4e5c5506 303 if (other.fPackages) {
304 fPackages = new TObjArray();
305 TIter next(other.fPackages);
306 TObject *obj;
307 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
308 fPackages->SetOwner();
309 }
c57f56b7 310 }
311 return *this;
312}
313
f965131e 314//______________________________________________________________________________
315void AliAnalysisAlien::AddIncludePath(const char *path)
316{
317// Add include path in the remote analysis macro.
318 TString p(path);
319 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
320 else fIncludePath += Form("-I%s ", path);
321}
322
c57f56b7 323//______________________________________________________________________________
324void AliAnalysisAlien::AddRunNumber(Int_t run)
325{
326// Add a run number to the list of runs to be processed.
327 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 328 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 329}
330
ee75cfc3 331//______________________________________________________________________________
332void AliAnalysisAlien::AddRunNumber(const char* run)
333{
334// Add a run number to the list of runs to be processed.
335 if (fRunNumbers.Length()) fRunNumbers += " ";
336 fRunNumbers += run;
337}
338
c57f56b7 339//______________________________________________________________________________
340void AliAnalysisAlien::AddDataFile(const char *lfn)
341{
342// Adds a data file to the input to be analysed. The file should be a valid LFN
343// or point to an existing file in the alien workdir.
344 if (!fInputFiles) fInputFiles = new TObjArray();
345 fInputFiles->Add(new TObjString(lfn));
346}
648174cf 347
348//______________________________________________________________________________
349void AliAnalysisAlien::AddExternalPackage(const char *package)
350{
351// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
352 if (fExternalPackages) fExternalPackages += " ";
353 fExternalPackages += package;
354}
355
c57f56b7 356//______________________________________________________________________________
357Bool_t AliAnalysisAlien::Connect()
358{
359// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
360 if (gGrid && gGrid->IsConnected()) return kTRUE;
361 if (!gSystem->Getenv("alien_API_USER")) {
362 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
363 gSystem->Getenv("UID"));
364 return kFALSE;
365 }
366 if (!gGrid) {
367 Info("Connect", "Trying to connect to AliEn ...");
368 TGrid::Connect("alien://");
369 }
370 if (!gGrid || !gGrid->IsConnected()) {
371 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
372 return kFALSE;
373 }
374 fUser = gGrid->GetUser();
375 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
376 return kTRUE;
377}
378
379//______________________________________________________________________________
380void AliAnalysisAlien::CdWork()
381{
382// Check validity of alien workspace. Create directory if possible.
383 if (!Connect()) {
384 Error("CdWork", "Alien connection required");
385 return;
386 }
387 TString homedir = gGrid->GetHomeDirectory();
388 TString workdir = homedir + fGridWorkingDir;
923e2ca5 389 if (DirectoryExists(workdir)) {
390 gGrid->Cd(workdir);
391 return;
392 }
393 // Work directory not existing - create it
394 gGrid->Cd(homedir);
395 if (gGrid->Mkdir(workdir)) {
396 gGrid->Cd(fGridWorkingDir);
397 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
398 } else {
399 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
400 workdir.Data(), homedir.Data());
401 fGridWorkingDir = "";
402 }
c57f56b7 403}
404
405//______________________________________________________________________________
406Bool_t AliAnalysisAlien::CheckInputData()
407{
408// Check validity of input data. If necessary, create xml files.
a8739e8a 409 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 410 if (!fGridDataDir.Length()) {
411 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
412 return kFALSE;
413 }
414 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
415 return kTRUE;
c57f56b7 416 }
417 // Process declared files
418 Bool_t is_collection = kFALSE;
419 Bool_t is_xml = kFALSE;
420 Bool_t use_tags = kFALSE;
421 Bool_t checked = kFALSE;
422 CdWork();
423 TString file;
424 TString workdir = gGrid->GetHomeDirectory();
425 workdir += fGridWorkingDir;
426 if (fInputFiles) {
427 TObjString *objstr;
428 TIter next(fInputFiles);
429 while ((objstr=(TObjString*)next())) {
430 file = workdir;
431 file += "/";
432 file += objstr->GetString();
433 // Store full lfn path
434 if (FileExists(file)) objstr->SetString(file);
435 else {
436 file = objstr->GetName();
437 if (!FileExists(objstr->GetName())) {
438 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
439 objstr->GetName(), workdir.Data());
440 return kFALSE;
441 }
442 }
443 Bool_t iscoll, isxml, usetags;
444 CheckDataType(file, iscoll, isxml, usetags);
445 if (!checked) {
446 checked = kTRUE;
447 is_collection = iscoll;
448 is_xml = isxml;
449 use_tags = usetags;
450 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
451 } else {
452 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
453 Error("CheckInputData", "Some conflict was found in the types of inputs");
454 return kFALSE;
455 }
456 }
457 }
458 }
459 // Process requested run numbers
a8739e8a 460 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 461 // Check validity of alien data directory
462 if (!fGridDataDir.Length()) {
463 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
464 return kFALSE;
465 }
923e2ca5 466 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 467 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
468 return kFALSE;
469 }
470 if (is_collection) {
471 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
472 return kFALSE;
473 }
474
475 if (checked && !is_xml) {
476 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
477 return kFALSE;
478 }
479 // Check validity of run number(s)
480 TObjArray *arr;
481 TObjString *os;
319593fb 482 Int_t nruns = 0;
904f9f5f 483 TString schunk, schunk2;
c57f56b7 484 TString path;
485 if (!checked) {
486 checked = kTRUE;
487 use_tags = fDataPattern.Contains("tag");
488 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
489 }
490 if (use_tags != fDataPattern.Contains("tag")) {
491 Error("CheckInputData", "Cannot mix input files using/not using tags");
492 return kFALSE;
493 }
494 if (fRunNumbers.Length()) {
a8739e8a 495 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 496 arr = fRunNumbers.Tokenize(" ");
497 TIter next(arr);
498 while ((os=(TObjString*)next())) {
499 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 500 if (!DirectoryExists(path)) {
501 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 502 continue;
c57f56b7 503 }
504 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
505 TString msg = "\n##### file: ";
506 msg += path;
507 msg += " type: xml_collection;";
508 if (use_tags) msg += " using_tags: Yes";
509 else msg += " using_tags: No";
510 Info("CheckDataType", msg.Data());
319593fb 511 if (fNrunsPerMaster<2) {
d2a409b2 512 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 513 } else {
514 nruns++;
515 if (((nruns-1)%fNrunsPerMaster) == 0) {
516 schunk = os->GetString();
517 }
518 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
519 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 520 AddDataFile(schunk);
319593fb 521 }
c57f56b7 522 }
523 delete arr;
a8739e8a 524 } else {
525 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
526 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 527 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 528 if (!DirectoryExists(path)) {
529// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 530 continue;
531 }
5fce53f4 532 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 533 TString msg = "\n##### file: ";
534 msg += path;
535 msg += " type: xml_collection;";
536 if (use_tags) msg += " using_tags: Yes";
537 else msg += " using_tags: No";
538 Info("CheckDataType", msg.Data());
319593fb 539 if (fNrunsPerMaster<2) {
5fce53f4 540 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 541 } else {
542 nruns++;
543 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 544 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 545 }
904f9f5f 546 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 547 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 548 schunk += schunk2;
d2a409b2 549 AddDataFile(schunk);
319593fb 550 }
a8739e8a 551 }
904f9f5f 552 if (!fInputFiles) {
553 schunk += schunk2;
554 AddDataFile(schunk);
555 }
c57f56b7 556 }
557 return kTRUE;
558}
559
560//______________________________________________________________________________
561Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
562{
563// Create dataset for the grid data directory + run number.
564 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
565 if (!Connect()) {
566 Error("CreateDataset", "Cannot create dataset with no grid connection");
567 return kFALSE;
568 }
569
570 // Cd workspace
571 CdWork();
572 TString workdir = gGrid->GetHomeDirectory();
573 workdir += fGridWorkingDir;
574
575 // Compose the 'find' command arguments
576 TString command;
577 TString options = "-x collection ";
bb885a9e 578 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 579 TString conditions = "";
580
581 TString file;
582 TString path;
319593fb 583 Int_t nruns = 0;
904f9f5f 584 TString schunk, schunk2;
ab254fd1 585 TGridCollection *cbase=0, *cadd=0;
d2a409b2 586 if (!fRunNumbers.Length() && !fRunRange[0]) {
587 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
588 // Make a single data collection from data directory.
589 path = fGridDataDir;
923e2ca5 590 if (!DirectoryExists(path)) {
d2a409b2 591 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
592 return kFALSE;
593 }
923e2ca5 594// CdWork();
d2a409b2 595 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
596 else file = Form("%s.xml", gSystem->BaseName(path));
0f389141 597 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
d2a409b2 598 command = "find ";
599 command += options;
600 command += path;
601 command += " ";
602 command += pattern;
603 command += conditions;
84fcd93f 604 printf("command: %s\n", command.Data());
d2a409b2 605 TGridResult *res = gGrid->Command(command);
606 if (res) delete res;
607 // Write standard output to file
608 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
0f389141 609 }
610 Bool_t fileExists = FileExists(file);
611 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
d2a409b2 612 // Copy xml file to alien space
0f389141 613 if (fileExists) gGrid->Rm(file);
d2a409b2 614 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
615 if (!FileExists(file)) {
616 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
617 return kFALSE;
618 }
619 // Update list of files to be processed.
620 }
621 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
622 return kTRUE;
623 }
c57f56b7 624 // Several runs
a8739e8a 625 if (fRunNumbers.Length()) {
626 TObjArray *arr = fRunNumbers.Tokenize(" ");
627 TObjString *os;
628 TIter next(arr);
629 while ((os=(TObjString*)next())) {
630 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 631 if (!DirectoryExists(path)) continue;
632// CdWork();
a8739e8a 633 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
634 else file = Form("%s.xml", os->GetString().Data());
319593fb 635 // If local collection file does not exist, create it via 'find' command.
636 if (gSystem->AccessPathName(file)) {
637 command = "find ";
638 command += options;
639 command += path;
640 command += pattern;
641 command += conditions;
642 TGridResult *res = gGrid->Command(command);
643 if (res) delete res;
644 // Write standard output to file
645 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
646 }
a8739e8a 647 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 648 // Check if there is one run per master job.
649 if (fNrunsPerMaster<2) {
650 if (FileExists(file)) {
0f389141 651 if (fOverwriteMode) gGrid->Rm(file);
652 else {
653 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
654 continue;
655 }
319593fb 656 }
657 // Copy xml file to alien space
658 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
659 if (!FileExists(file)) {
660 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
661 delete arr;
662 return kFALSE;
663 }
664 } else {
665 nruns++;
666 if (((nruns-1)%fNrunsPerMaster) == 0) {
667 schunk = os->GetString();
668 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
669 } else {
670 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 671 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 672 cbase->Add(cadd);
673 delete cadd;
674 }
675 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
676 continue;
677 }
678 schunk += Form("_%s.xml", os->GetString().Data());
0f389141 679 if (FileExists(schunk)) {
680 if (fOverwriteMode) gGrid->Rm(file);
681 else {
682 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
683 continue;
684 }
319593fb 685 }
84fcd93f 686 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 687 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 688 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 689 if (!FileExists(schunk)) {
690 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
691 delete arr;
692 return kFALSE;
693 }
694 }
a8739e8a 695 }
696 delete arr;
697 } else {
698 // Process a full run range.
699 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 700 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 701 if (!DirectoryExists(path)) continue;
702// CdWork();
a8739e8a 703 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 704 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
0f389141 705 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
706 if (fOverwriteMode) gGrid->Rm(file);
707 else {
708 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
709 continue;
710 }
a8739e8a 711 }
319593fb 712 // If local collection file does not exist, create it via 'find' command.
0f389141 713 if (gSystem->AccessPathName(file) || fOverwriteMode) {
319593fb 714 command = "find ";
715 command += options;
716 command += path;
717 command += pattern;
718 command += conditions;
719 TGridResult *res = gGrid->Command(command);
720 if (res) delete res;
721 // Write standard output to file
722 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
723 }
a8739e8a 724 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 725 // Check if there is one run per master job.
726 if (fNrunsPerMaster<2) {
727 if (FileExists(file)) {
0f389141 728 if (fOverwriteMode) gGrid->Rm(file);
729 else {
730 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
731 continue;
732 }
319593fb 733 }
734 // Copy xml file to alien space
735 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
736 if (!FileExists(file)) {
737 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
738 return kFALSE;
739 }
740 } else {
741 nruns++;
95e5b448 742 // Check if the collection for the chunk exist locally.
743 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
0f389141 744 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
745 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
746 else continue;
747 }
84fcd93f 748 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 749 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 750 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 751 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
752 } else {
753 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
754 cbase->Add(cadd);
755 delete cadd;
756 }
904f9f5f 757 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
758 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 759 continue;
760 }
904f9f5f 761 schunk = schunk2;
319593fb 762 if (FileExists(schunk)) {
0f389141 763 if (fOverwriteMode) gGrid->Rm(schunk);
764 else {
765 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
766 continue;
767 }
319593fb 768 }
84fcd93f 769 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 770 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 771 if (FileExists(schunk)) {
0f389141 772 if (fOverwriteMode) gGrid->Rm(schunk);
773 else {
774 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
775 continue;
776 }
95e5b448 777 }
319593fb 778 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
779 if (!FileExists(schunk)) {
780 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
781 return kFALSE;
782 }
783 }
c57f56b7 784 }
a8739e8a 785 }
c57f56b7 786 return kTRUE;
787}
788
789//______________________________________________________________________________
790Bool_t AliAnalysisAlien::CreateJDL()
791{
792// Generate a JDL file according to current settings. The name of the file is
793// specified by fJDLName.
794 Bool_t error = kFALSE;
795 TObjArray *arr = 0;
796 Bool_t copy = kTRUE;
797 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
798 Bool_t generate = kTRUE;
799 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
800 if (!Connect()) {
801 Error("CreateJDL", "Alien connection required");
802 return kFALSE;
803 }
804 // Check validity of alien workspace
805 CdWork();
806 TString workdir = gGrid->GetHomeDirectory();
807 workdir += fGridWorkingDir;
808 if (generate) {
809 TObjString *os;
810 if (!fInputFiles) {
811 Error("CreateJDL()", "Define some input files for your analysis.");
812 error = kTRUE;
813 }
814 // Compose list of input files
815 // Check if output files were defined
816 if (!fOutputFiles.Length()) {
817 Error("CreateJDL", "You must define at least one output file");
818 error = kTRUE;
819 }
820 // Check if an output directory was defined and valid
821 if (!fGridOutputDir.Length()) {
822 Error("CreateJDL", "You must define AliEn output directory");
823 error = kTRUE;
824 } else {
d2a409b2 825 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 826 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 827 if (gGrid->Mkdir(fGridOutputDir)) {
828 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
829 } else {
830 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 831 // error = kTRUE;
c57f56b7 832 }
833 }
834 gGrid->Cd(workdir);
835 }
836 // Exit if any error up to now
837 if (error) return kFALSE;
838 // Set JDL fields
0f389141 839 if (!fUser.IsNull()) {
840 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
841 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
842 }
843 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
844 TString mergeExec = fExecutable;
845 mergeExec.ReplaceAll(".sh", "_merge.sh");
846 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
847 mergeExec.ReplaceAll(".sh", ".C");
848 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
b5b9dee8 849 if (!fArguments.IsNull())
850 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
0f389141 851 fMergingJDL->SetArguments("$1");
852 fGridJDL->SetTTL((UInt_t)fTTL);
853 fMergingJDL->SetTTL((UInt_t)fTTL);
854 if (fMaxInitFailed > 0) {
c57f56b7 855 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
0f389141 856 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
857 }
858 if (fSplitMaxInputFileNumber > 0) {
c57f56b7 859 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
0f389141 860 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
861 }
862 if (fSplitMode.Length()) {
c57f56b7 863 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
0f389141 864 fGridJDL->SetDescription("Split", "We split per SE or file");
865 }
866 if (!fAliROOTVersion.IsNull()) {
867 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
868 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
869 }
870 if (!fROOTVersion.IsNull()) {
c57f56b7 871 fGridJDL->AddToPackages("ROOT", fROOTVersion);
0f389141 872 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
873 }
874 if (!fAPIVersion.IsNull()) {
c57f56b7 875 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
0f389141 876 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
877 }
648174cf 878 if (!fExternalPackages.IsNull()) {
879 arr = fExternalPackages.Tokenize(" ");
880 TIter next(arr);
881 while ((os=(TObjString*)next())) {
882 TString pkgname = os->GetString();
883 Int_t index = pkgname.Index("::");
884 TString pkgversion = pkgname(index+2, pkgname.Length());
885 pkgname.Remove(index);
886 fGridJDL->AddToPackages(pkgname, pkgversion);
0f389141 887 fMergingJDL->AddToPackages(pkgname, pkgversion);
648174cf 888 }
889 delete arr;
890 }
0f389141 891 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
892 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
893 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
f10e8481 894 TString analysisFile = fExecutable;
895 analysisFile.ReplaceAll(".sh", ".root");
896 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
f866cba5 897 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 898 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
899 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
900 if (fAdditionalLibs.Length()) {
901 arr = fAdditionalLibs.Tokenize(" ");
902 TIter next(arr);
903 while ((os=(TObjString*)next())) {
904 if (os->GetString().Contains(".so")) continue;
905 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
0f389141 906 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
c57f56b7 907 }
908 delete arr;
909 }
4e5c5506 910 if (fPackages) {
911 TIter next(fPackages);
912 TObject *obj;
0f389141 913 while ((obj=next())) {
4e5c5506 914 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
0f389141 915 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
916 }
4e5c5506 917 }
c57f56b7 918 if (fOutputArchive.Length()) {
919 arr = fOutputArchive.Tokenize(" ");
920 TIter next(arr);
0f389141 921 Bool_t first = kTRUE;
922 const char *comment = "Files to be archived";
923 const char *comment1 = comment;
924 while ((os=(TObjString*)next())) {
925 if (!first) comment = NULL;
926 if (!os->GetString().Contains("@") && fCloseSE.Length())
927 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
928 else
929 fGridJDL->AddToOutputArchive(os->GetString(), comment);
930 first = kFALSE;
931 }
c57f56b7 932 delete arr;
0f389141 933 TString outputArchive = fOutputArchive;
934 if (!fMergeExcludes.IsNull()) {
935 arr = fMergeExcludes.Tokenize(" ");
705adb3e 936 TIter next1(arr);
937 while ((os=(TObjString*)next1())) {
0f389141 938 outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
939 outputArchive.ReplaceAll(os->GetString(),"");
940 }
941 delete arr;
942 }
943 arr = outputArchive.Tokenize(" ");
705adb3e 944 TIter next2(arr);
0f389141 945 comment = comment1;
946 first = kTRUE;
705adb3e 947 while ((os=(TObjString*)next2())) {
0f389141 948 if (!first) comment = NULL;
949 if (!os->GetString().Contains("@") && fCloseSE.Length())
950 fMergingJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
951 else
952 fMergingJDL->AddToOutputArchive(os->GetString(), comment);
953 first = kFALSE;
954 }
955 delete arr;
c57f56b7 956 }
c57f56b7 957 arr = fOutputFiles.Tokenize(" ");
958 TIter next(arr);
0f389141 959 Bool_t first = kTRUE;
960 const char *comment = "Files to be archived";
961 const char *comment1 = comment;
43da816a 962 while ((os=(TObjString*)next())) {
e1eaf596 963 // Ignore ouputs in jdl that are also in outputarchive
964 TString sout = os->GetString();
965 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
966 if (fOutputArchive.Contains(sout)) continue;
0f389141 967 if (!first) comment = NULL;
43da816a 968 if (!os->GetString().Contains("@") && fCloseSE.Length())
0f389141 969 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
43da816a 970 else
0f389141 971 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
972 first = kFALSE;
43da816a 973 }
c57f56b7 974 delete arr;
0f389141 975 if (fOutputFiles.Length()) {
976 TString outputFiles = fOutputFiles;
977 if (!fMergeExcludes.IsNull()) {
978 arr = fMergeExcludes.Tokenize(" ");
705adb3e 979 TIter next1(arr);
980 while ((os=(TObjString*)next1())) {
0f389141 981 outputFiles.ReplaceAll(Form("%s,",os->GetString().Data()),"");
982 outputFiles.ReplaceAll(os->GetString(),"");
983 }
984 delete arr;
985 }
986 arr = outputFiles.Tokenize(" ");
705adb3e 987 TIter next2(arr);
0f389141 988 comment = comment1;
989 first = kTRUE;
705adb3e 990 while ((os=(TObjString*)next2())) {
0f389141 991 // Ignore ouputs in jdl that are also in outputarchive
992 TString sout = os->GetString();
993 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
994 if (fOutputArchive.Contains(sout)) continue;
995 if (!first) comment = NULL;
996 if (!os->GetString().Contains("@") && fCloseSE.Length())
997 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
998 else
999 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
1000 }
1001 delete arr;
1002 }
1003 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1004 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
0d5d317c 1005 TString validationScript = fExecutable;
1006 validationScript.ReplaceAll(".sh", "_validation.sh");
0f389141 1007 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1008 validationScript = fExecutable;
1009 validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
1010 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1011 if (fMasterResubmitThreshold) {
1012 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1013 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1014 }
d2a409b2 1015 // Write a jdl with 2 input parameters: collection name and output dir name.
1016 WriteJDL(copy);
c57f56b7 1017 }
1018 // Copy jdl to grid workspace
a8739e8a 1019 if (copy) {
b5e4aaa7 1020 // Check if an output directory was defined and valid
1021 if (!fGridOutputDir.Length()) {
1022 Error("CreateJDL", "You must define AliEn output directory");
1023 return kFALSE;
1024 } else {
1025 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
bb2e67a0 1026 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
b5e4aaa7 1027 if (gGrid->Mkdir(fGridOutputDir)) {
1028 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1029 } else {
1030 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1031 return kFALSE;
1032 }
1033 }
1034 gGrid->Cd(workdir);
1035 }
648174cf 1036 if (TestBit(AliAnalysisGrid::kSubmit)) {
0f389141 1037 TString mergeJDLName = fExecutable;
1038 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
648174cf 1039 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1040 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1041 if (fProductionMode) {
648174cf 1042 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1043 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1044 }
648174cf 1045 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1046 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1047 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
648174cf 1048 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1049 if (fMergeViaJDL) {
1050 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1051 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1052 }
648174cf 1053 }
c57f56b7 1054 if (fAdditionalLibs.Length()) {
e7c71df0 1055 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 1056 TObjString *os;
1057 TIter next(arr);
1058 while ((os=(TObjString*)next())) {
c57f56b7 1059 if (os->GetString().Contains(".so")) continue;
4e5c5506 1060 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 1061 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1062 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1063 }
1064 delete arr;
1065 }
4e5c5506 1066 if (fPackages) {
1067 TIter next(fPackages);
1068 TObject *obj;
1069 while ((obj=next())) {
fdbbc7be 1070 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
4e5c5506 1071 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1072 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1073 }
1074 }
c57f56b7 1075 }
1076 return kTRUE;
1077}
1078
a8739e8a 1079//______________________________________________________________________________
d2a409b2 1080Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 1081{
1082// Writes one or more JDL's corresponding to findex. If findex is negative,
1083// all run numbers are considered in one go (jdl). For non-negative indices
1084// they correspond to the indices in the array fInputFiles.
1085 if (!fInputFiles) return kFALSE;
1086 TObjString *os;
a8739e8a 1087 TString workdir = gGrid->GetHomeDirectory();
1088 workdir += fGridWorkingDir;
d2a409b2 1089
1090 if (!fRunNumbers.Length() && !fRunRange[0]) {
1091 // One jdl with no parameters in case input data is specified by name.
a8739e8a 1092 TIter next(fInputFiles);
1093 while ((os=(TObjString*)next()))
0f389141 1094 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()), "Input xml collections");
648174cf 1095 if (!fOutputSingle.IsNull())
0f389141 1096 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1097 else {
1098 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1099 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1100 }
a8739e8a 1101 } else {
d2a409b2 1102 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
0f389141 1103 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
cd11251e 1104 if (!fOutputSingle.IsNull()) {
0f389141 1105 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1106 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
cd11251e 1107 } else {
0f389141 1108 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1109 fMergingJDL->SetOutputDirectory(Form("%s/$1", fGridOutputDir.Data()), "Output directory");
cd11251e 1110 }
a8739e8a 1111 }
1112
1113
1114 // Generate the JDL as a string
1115 TString sjdl = fGridJDL->Generate();
0f389141 1116 TString sjdl1 = fMergingJDL->Generate();
a8739e8a 1117 Int_t index;
a8739e8a 1118 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1119 sjdl.ReplaceAll("(member", "\n (member");
1120 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1121 sjdl.ReplaceAll("{", "{\n ");
1122 sjdl.ReplaceAll("};", "\n};");
1123 sjdl.ReplaceAll("{\n \n", "{\n");
1124 sjdl.ReplaceAll("\n\n", "\n");
1125 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
0f389141 1126 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1127 sjdl1.ReplaceAll("(member", "\n (member");
1128 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1129 sjdl1.ReplaceAll("{", "{\n ");
1130 sjdl1.ReplaceAll("};", "\n};");
1131 sjdl1.ReplaceAll("{\n \n", "{\n");
1132 sjdl1.ReplaceAll("\n\n", "\n");
1133 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
a8739e8a 1134 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1135 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1136 index = sjdl.Index("JDLVariables");
1137 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
0f389141 1138 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1139 sjdl1.Prepend(Form("Jobtag = {\n \"comment:Merging_%s\"\n};\n", fJobTag.Data()));
1140 index = sjdl1.Index("JDLVariables");
1141 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
a8739e8a 1142 // Write jdl to file
a8739e8a 1143 ofstream out;
d2a409b2 1144 out.open(fJDLName.Data(), ios::out);
a8739e8a 1145 if (out.bad()) {
d2a409b2 1146 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1147 return kFALSE;
1148 }
1149 out << sjdl << endl;
0f389141 1150 TString mergeJDLName = fExecutable;
1151 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1152 if (fMergeViaJDL) {
1153 ofstream out1;
1154 out1.open(mergeJDLName.Data(), ios::out);
1155 if (out.bad()) {
1156 Error("CreateJDL", "Bad file name: %s", mergeJDLName.Data());
1157 return kFALSE;
1158 }
1159 out1 << sjdl1 << endl;
1160 }
a8739e8a 1161
1162 // Copy jdl to grid workspace
1163 if (!copy) {
1164 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1165 } else {
d2a409b2 1166 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
0f389141 1167 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1168 if (fProductionMode) {
b5fe9cba 1169 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
0f389141 1170 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1171 }
d2a409b2 1172 if (FileExists(locjdl)) gGrid->Rm(locjdl);
0f389141 1173 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1174 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
5513444a 1175 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
0f389141 1176 if (fMergeViaJDL) {
1177 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1178 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1179 }
a8739e8a 1180 }
1181 return kTRUE;
1182}
1183
c57f56b7 1184//______________________________________________________________________________
5513444a 1185Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1186{
1187// Returns true if file exists.
5513444a 1188 if (!gGrid) return kFALSE;
c57f56b7 1189 TGridResult *res = gGrid->Ls(lfn);
1190 if (!res) return kFALSE;
1191 TMap *map = dynamic_cast<TMap*>(res->At(0));
1192 if (!map) {
1193 delete res;
1194 return kFALSE;
1195 }
1196 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1197 if (!objs || !objs->GetString().Length()) {
1198 delete res;
1199 return kFALSE;
1200 }
1201 delete res;
1202 return kTRUE;
1203}
1204
923e2ca5 1205//______________________________________________________________________________
1206Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1207{
1208// Returns true if directory exists. Can be also a path.
1209 if (!gGrid) return kFALSE;
1210 // Check if dirname is a path
1211 TString dirstripped = dirname;
1212 dirstripped = dirstripped.Strip();
1213 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1214 TString dir = gSystem->BaseName(dirstripped);
1215 dir += "/";
1216 TString path = gSystem->DirName(dirstripped);
1217 TGridResult *res = gGrid->Ls(path, "-F");
1218 if (!res) return kFALSE;
1219 TIter next(res);
1220 TMap *map;
1221 TObject *obj;
1222 while ((map=dynamic_cast<TMap*>(next()))) {
1223 obj = map->GetValue("name");
1224 if (!obj) break;
1225 if (dir == obj->GetName()) {
1226 delete res;
1227 return kTRUE;
1228 }
1229 }
1230 delete res;
1231 return kFALSE;
1232}
1233
c57f56b7 1234//______________________________________________________________________________
1235void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1236{
1237// Check input data type.
1238 is_collection = kFALSE;
1239 is_xml = kFALSE;
1240 use_tags = kFALSE;
1241 if (!gGrid) {
1242 Error("CheckDataType", "No connection to grid");
1243 return;
1244 }
1245 is_collection = IsCollection(lfn);
1246 TString msg = "\n##### file: ";
1247 msg += lfn;
1248 if (is_collection) {
1249 msg += " type: raw_collection;";
1250 // special treatment for collections
1251 is_xml = kFALSE;
1252 // check for tag files in the collection
1253 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1254 if (!res) {
1255 msg += " using_tags: No (unknown)";
1256 Info("CheckDataType", msg.Data());
1257 return;
1258 }
1259 const char* typeStr = res->GetKey(0, "origLFN");
1260 if (!typeStr || !strlen(typeStr)) {
1261 msg += " using_tags: No (unknown)";
1262 Info("CheckDataType", msg.Data());
1263 return;
1264 }
1265 TString file = typeStr;
1266 use_tags = file.Contains(".tag");
1267 if (use_tags) msg += " using_tags: Yes";
1268 else msg += " using_tags: No";
1269 Info("CheckDataType", msg.Data());
1270 return;
1271 }
1272 TString slfn(lfn);
1273 slfn.ToLower();
1274 is_xml = slfn.Contains(".xml");
1275 if (is_xml) {
1276 // Open xml collection and check if there are tag files inside
1277 msg += " type: xml_collection;";
1278 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1279 if (!coll) {
1280 msg += " using_tags: No (unknown)";
1281 Info("CheckDataType", msg.Data());
1282 return;
1283 }
1284 TMap *map = coll->Next();
1285 if (!map) {
1286 msg += " using_tags: No (unknown)";
1287 Info("CheckDataType", msg.Data());
1288 return;
1289 }
1290 map = (TMap*)map->GetValue("");
1291 TString file;
1292 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1293 use_tags = file.Contains(".tag");
1294 delete coll;
1295 if (use_tags) msg += " using_tags: Yes";
1296 else msg += " using_tags: No";
1297 Info("CheckDataType", msg.Data());
1298 return;
1299 }
1300 use_tags = slfn.Contains(".tag");
1301 if (slfn.Contains(".root")) msg += " type: root file;";
f866cba5 1302 else msg += " type: unknown file;";
c57f56b7 1303 if (use_tags) msg += " using_tags: Yes";
1304 else msg += " using_tags: No";
1305 Info("CheckDataType", msg.Data());
1306}
1307
4e5c5506 1308//______________________________________________________________________________
1309void AliAnalysisAlien::EnablePackage(const char *package)
1310{
1311// Enables a par file supposed to exist in the current directory.
1312 TString pkg(package);
1313 pkg.ReplaceAll(".par", "");
1314 pkg += ".par";
1315 if (gSystem->AccessPathName(pkg)) {
ebcdf05e 1316 Fatal("EnablePackage", "Package %s not found", pkg.Data());
4e5c5506 1317 return;
1318 }
1319 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1320 Info("EnablePackage", "AliEn plugin will use .par packages");
1321 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1322 if (!fPackages) {
1323 fPackages = new TObjArray();
1324 fPackages->SetOwner();
1325 }
1326 fPackages->Add(new TObjString(pkg));
1327}
1328
d2a409b2 1329//______________________________________________________________________________
1330const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1331{
1332// Get job status for all jobs with jobid>jobidstart.
1333 static char mstatus[20];
1334 mstatus[0] = '\0';
1335 nrunning = 0;
1336 nwaiting = 0;
1337 nerror = 0;
1338 ndone = 0;
1339 TGridJobStatusList *list = gGrid->Ps("");
1340 if (!list) return mstatus;
1341 Int_t nentries = list->GetSize();
1342 TGridJobStatus *status;
1343 Int_t pid;
1344 for (Int_t ijob=0; ijob<nentries; ijob++) {
1345 status = (TGridJobStatus *)list->At(ijob);
1346 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1347 if (pid<jobidstart) continue;
1348 if (pid == lastid) {
1349 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1350 }
1351 switch (status->GetStatus()) {
1352 case TGridJobStatus::kWAITING:
1353 nwaiting++; break;
1354 case TGridJobStatus::kRUNNING:
1355 nrunning++; break;
1356 case TGridJobStatus::kABORTED:
1357 case TGridJobStatus::kFAIL:
1358 case TGridJobStatus::kUNKNOWN:
1359 nerror++; break;
1360 case TGridJobStatus::kDONE:
1361 ndone++;
1362 }
1363 }
1364 list->Delete();
1365 delete list;
1366 return mstatus;
1367}
1368
c57f56b7 1369//______________________________________________________________________________
1370Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1371{
1372// Returns true if file is a collection. Functionality duplicated from
1373// TAlien::Type() because we don't want to directly depend on TAlien.
1374 if (!gGrid) {
1375 Error("IsCollection", "No connection to grid");
1376 return kFALSE;
1377 }
1378 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1379 if (!res) return kFALSE;
1380 const char* typeStr = res->GetKey(0, "type");
1381 if (!typeStr || !strlen(typeStr)) return kFALSE;
1382 if (!strcmp(typeStr, "collection")) return kTRUE;
1383 delete res;
1384 return kFALSE;
1385}
1386
fe2d7fc2 1387//______________________________________________________________________________
1388Bool_t AliAnalysisAlien::IsSingleOutput() const
1389{
1390// Check if single-ouput option is on.
1391 return (!fOutputSingle.IsNull());
1392}
1393
16a4353c 1394//______________________________________________________________________________
1395void AliAnalysisAlien::Print(Option_t *) const
1396{
1397// Print current plugin settings.
84fcd93f 1398 printf("### AliEn analysis plugin current settings ###\n");
1399 printf("= Production mode:______________________________ %d\n", fProductionMode);
1400 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1401 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1402 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1403 if (fUser.Length())
84fcd93f 1404 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1405 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1406 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1407 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1408 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1409 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1410 if (fRunNumbers.Length())
84fcd93f 1411 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1412 if (fRunRange[0])
5fce53f4 1413 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1414 if (!fRunRange[0] && !fRunNumbers.Length()) {
1415 TIter next(fInputFiles);
1416 TObject *obj;
1417 TString list;
1418 while ((obj=next())) list += obj->GetName();
84fcd93f 1419 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1420 }
1421 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1422 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1423 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1424 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1425 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1426 printf("=====================================================================\n");
1427 printf("= Job price: ___________________________________ %d\n", fPrice);
1428 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1429 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1430 if (fMaxInitFailed>0)
84fcd93f 1431 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1432 if (fMasterResubmitThreshold>0)
84fcd93f 1433 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1434 if (fNrunsPerMaster>0)
84fcd93f 1435 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1436 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1437 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1438 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1439 if (fArguments.Length())
84fcd93f 1440 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1441 if (fExecutableArgs.Length())
1442 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1443 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1444 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1445 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1446 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1447 if (fDatasetName)
84fcd93f 1448 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1449 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1450 if (fIncludePath.Data())
84fcd93f 1451 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1452 if (fCloseSE.Length())
84fcd93f 1453 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1454 if (fFriendChainName.Length())
84fcd93f 1455 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1456 if (fPackages) {
1457 TIter next(fPackages);
1458 TObject *obj;
1459 TString list;
1460 while ((obj=next())) list += obj->GetName();
84fcd93f 1461 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1462 }
1463}
1464
c57f56b7 1465//______________________________________________________________________________
1466void AliAnalysisAlien::SetDefaults()
1467{
1468// Set default values for everything. What cannot be filled will be left empty.
1469 if (fGridJDL) delete fGridJDL;
1470 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
0f389141 1471 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
c57f56b7 1472 fPrice = 1;
1473 fTTL = 30000;
1474 fSplitMaxInputFileNumber = 100;
1475 fMaxInitFailed = 0;
1476 fMasterResubmitThreshold = 0;
bb885a9e 1477 fNtestFiles = 10;
a8739e8a 1478 fRunRange[0] = 0;
1479 fRunRange[1] = 0;
319593fb 1480 fNrunsPerMaster = 1;
16a4353c 1481 fMaxMergeFiles = 100;
c57f56b7 1482 fRunNumbers = "";
1483 fExecutable = "analysis.sh";
0a1c1f7f 1484 fExecutableCommand = "root -b -q";
c57f56b7 1485 fArguments = "";
631c0b05 1486 fExecutableArgs = "";
c57f56b7 1487 fAnalysisMacro = "myAnalysis.C";
1488 fAnalysisSource = "";
1489 fAdditionalLibs = "";
1490 fSplitMode = "se";
1491 fAPIVersion = "";
1492 fROOTVersion = "";
1493 fAliROOTVersion = "";
1494 fUser = ""; // Your alien user name
1495 fGridWorkingDir = "";
1496 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1497 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1498 fFriendChainName = "";
c57f56b7 1499 fGridOutputDir = "output";
1500 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1501 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1502 fInputFormat = "xml-single";
1503 fJDLName = "analysis.jdl";
c6cb3634 1504 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1505 fMergeExcludes = "";
0f389141 1506 fMergeViaJDL = 0;
c57f56b7 1507}
1508
0f389141 1509//______________________________________________________________________________
1510Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge)
1511{
1512// Merge all registered outputs from basedir.
1513 TString output_file = output;
1514 TString command;
1515 TString output_chunk;
1516 TString previous_chunk = "";
1517 Int_t count_chunk = 0;
1518 Int_t count_zero = nmaxmerge;
1519 Bool_t merged = kTRUE;
1520 Int_t index = output_file.Index("@");
1521 if (index > 0) output_file.Remove(index);
1522 command = Form("find %s/ *%s", basedir, output_file.Data());
1523 printf("command: %s\n", command.Data());
1524 TGridResult *res = gGrid->Command(command);
1525 if (!res) {
1526 printf("Error: No result for the find command\n");
1527 return kFALSE;
1528 }
1529
1530 TFileMerger *fm = 0;
1531 TIter nextmap(res);
1532 TMap *map = 0;
1533 // Check if there is a merge operation to resume
1534 output_chunk = output_file;
1535 output_chunk.ReplaceAll(".root", "_*.root");
1536 // Check for existent temporary merge files
1537 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1538 while (1) {
1539 // Skip as many input files as in a chunk
1540 for (Int_t counter=0; counter<nmaxmerge; counter++) map = (TMap*)nextmap();
1541 if (!map) {
1542 ::Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1543 delete res;
1544 return kFALSE;
1545 }
1546 output_chunk = output_file;
1547 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1548 count_chunk++;
1549 if (gSystem->AccessPathName(output_chunk)) continue;
1550 // Merged file with chunks up to <count_chunk> found
1551 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1552 previous_chunk = output_chunk;
1553 break;
1554 }
1555 }
1556 count_zero = nmaxmerge;
1557
1558 while ((map=(TMap*)nextmap())) {
1559 // Loop 'find' results and get next LFN
1560 if (count_zero == nmaxmerge) {
1561 // First file in chunk - create file merger and add previous chunk if any.
1562 fm = new TFileMerger(kFALSE);
1563 fm->SetFastMethod(kTRUE);
1564 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1565 output_chunk = output_file;
1566 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1567 }
1568 // If last file found, put merged results in the output file
1569 if (map == res->Last()) output_chunk = output_file;
1570 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1571 if (!objs || !objs->GetString().Length()) {
1572 // Nothing found - skip this output
1573 delete res;
1574 delete fm;
1575 return kFALSE;
1576 }
1577 // Add file to be merged and decrement chunk counter.
1578 fm->AddFile(objs->GetString());
1579 count_zero--;
1580 if (count_zero==0 || map == res->Last()) {
1581 fm->OutputFile(output_chunk);
1582 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1583 // Nothing found - skip this output
1584 ::Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1585 delete res;
1586 delete fm;
1587 return kFALSE;
1588 }
1589 // Merge the outputs, then go to next chunk
1590 if (!fm->Merge()) {
1591 ::Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1592 delete res;
1593 delete fm;
1594 merged = kFALSE;
1595 return kFALSE;
1596 } else {
1597 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
1598 gSystem->Unlink(previous_chunk);
1599 }
1600 if (map == res->Last()) {
1601 delete res;
1602 delete fm;
1603 break;
1604 }
1605 count_chunk++;
1606 count_zero = nmaxmerge;
1607 previous_chunk = output_chunk;
1608 }
1609 }
1610 return merged;
1611}
1612
c57f56b7 1613//______________________________________________________________________________
1614Bool_t AliAnalysisAlien::MergeOutputs()
1615{
1616// Merge analysis outputs existing in the AliEn space.
1617 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1618 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1619 if (!Connect()) {
1620 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1621 return kFALSE;
0f389141 1622 }
1623 if (fMergeViaJDL && !fProductionMode && TestBit(AliAnalysisGrid::kMerge)) {
1624 Info("MergeOutputs", "Submitting merging JDL");
1625 SubmitMerging();
1626 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
001cb79e 1627 Info("MergeOutputs", "### Not executing Terminate for now. This is not an error.");
1628 return kFALSE;
c57f56b7 1629 }
1630 // Get the output path
d2a409b2 1631 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1632 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1633 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1634 return kFALSE;
1635 }
1636 if (!fOutputFiles.Length()) {
1637 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1638 return kFALSE;
0f389141 1639 }
1640 // Check if fast read option was requested
1641 if (fFastReadOption) {
1642 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !");
1643 gEnv->SetValue("XNet.ConnectTimeout",5);
1644 gEnv->SetValue("XNet.RequestTimeout",5);
1645 gEnv->SetValue("XNet.MaxRedirectCount",2);
1646 gEnv->SetValue("XNet.ReconnectTimeout",5);
1647 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
c57f56b7 1648 }
1649 TObjArray *list = fOutputFiles.Tokenize(" ");
1650 TIter next(list);
1651 TObjString *str;
c57f56b7 1652 TString output_file;
1653 Bool_t merged = kTRUE;
1654 while((str=(TObjString*)next())) {
1655 output_file = str->GetString();
1656 Int_t index = output_file.Index("@");
1657 if (index > 0) output_file.Remove(index);
319593fb 1658 // Skip already merged outputs
1659 if (!gSystem->AccessPathName(output_file)) {
1660 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1661 continue;
1662 }
bb885a9e 1663 if (fMergeExcludes.Length() &&
1664 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1665 // Perform a 'find' command in the output directory, looking for registered outputs
0f389141 1666 merged = MergeOutput(output_file, fGridOutputDir, fMaxMergeFiles);
1667 if (!merged) {
1668 Error("MergeOutputs", "Terminate() will NOT be executed");
1669 return kFALSE;
1670 }
c57f56b7 1671 }
0f389141 1672 return kTRUE;
c57f56b7 1673}
1674
bb885a9e 1675//______________________________________________________________________________
1676void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1677{
1678// Use the output files connected to output containers from the analysis manager
1679// rather than the files defined by SetOutputFiles
1680 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1681 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1682 analysis manager");
1683 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1684}
1685
5513444a 1686//______________________________________________________________________________
1687Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1688{
1689// Start remote grid analysis.
1690
43da816a 1691 // Check if output files have to be taken from the analysis manager
1692 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1693 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1694 if (!mgr || !mgr->IsInitialized()) {
1695 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1696 return kFALSE;
43da816a 1697 }
1698 fOutputFiles = "";
1699 TIter next(mgr->GetOutputs());
1700 AliAnalysisDataContainer *output;
1701 while ((output=(AliAnalysisDataContainer*)next())) {
1702 const char *filename = output->GetFileName();
1703 if (!(strcmp(filename, "default"))) {
1704 if (!mgr->GetOutputEventHandler()) continue;
1705 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1706 }
84fcd93f 1707 if (fOutputFiles.Contains(filename)) continue;
43da816a 1708 if (fOutputFiles.Length()) fOutputFiles += " ";
1709 fOutputFiles += filename;
1710 }
c07b9ce2 1711 // Add extra files registered to the analysis manager
1712 if (mgr->GetExtraFiles().Length()) {
1713 if (fOutputFiles.Length()) fOutputFiles += " ";
1714 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1715 }
43da816a 1716 }
f7b1cbc2 1717// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1718 if (TestBit(AliAnalysisGrid::kOffline)) {
1719 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1720 \n there nor any job run. You can revise the JDL and analysis \
1721 \n macro then run the same in \"submit\" mode.");
1722 } else if (TestBit(AliAnalysisGrid::kTest)) {
1723 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1724 \n dataset.");
1725 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1726 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1727 \n space and job submitted.");
1728 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1729 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
0f389141 1730 if (fMergeViaJDL) CheckInputData();
5513444a 1731 return kTRUE;
c57f56b7 1732 } else {
1733 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1734 }
1735
1736 if (!Connect()) {
1737 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1738 return kFALSE;
16a4353c 1739 }
1740 Print();
c57f56b7 1741 if (!CheckInputData()) {
1742 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1743 return kFALSE;
c57f56b7 1744 }
1745 CreateDataset(fDataPattern);
1746 WriteAnalysisFile();
1747 WriteAnalysisMacro();
1748 WriteExecutable();
1749 WriteValidationScript();
0f389141 1750 if (fMergeViaJDL) {
1751 WriteMergingMacro();
1752 WriteMergeExecutable();
1753 WriteValidationScript(kTRUE);
1754 }
5513444a 1755 if (!CreateJDL()) return kFALSE;
1756 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1757 if (TestBit(AliAnalysisGrid::kTest)) {
1758 // Locally testing the analysis
1759 Info("StartAnalysis", "\n_______________________________________________________________________ \
1760 \n Running analysis script in a daughter shell as on a worker node \
1761 \n_______________________________________________________________________");
1762 TObjArray *list = fOutputFiles.Tokenize(" ");
1763 TIter next(list);
1764 TObjString *str;
1765 TString output_file;
1766 while((str=(TObjString*)next())) {
1767 output_file = str->GetString();
1768 Int_t index = output_file.Index("@");
1769 if (index > 0) output_file.Remove(index);
43da816a 1770 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1771 }
1772 delete list;
1773 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1774 TString validationScript = fExecutable;
1775 validationScript.ReplaceAll(".sh", "_validation.sh");
1776 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1777// gSystem->Exec("cat stdout");
5513444a 1778 return kFALSE;
c57f56b7 1779 }
5513444a 1780 // Check if submitting is managed by LPM manager
a3e84053 1781 if (fProductionMode) {
5513444a 1782 TString prodfile = fJDLName;
1783 prodfile.ReplaceAll(".jdl", ".prod");
1784 WriteProductionFile(prodfile);
1785 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1786 return kFALSE;
1787 }
a8739e8a 1788 // Submit AliEn job(s)
d2a409b2 1789 gGrid->Cd(fGridOutputDir);
a8739e8a 1790 TGridResult *res;
c57f56b7 1791 TString jobID = "";
d2a409b2 1792 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1793 // Submit a given xml or a set of runs
a8739e8a 1794 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1795 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1796 if (res) {
1797 const char *cjobId = res->GetKey(0,"jobId");
1798 if (!cjobId) {
1799 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1800 return kFALSE;
a8739e8a 1801 } else {
1802 Info("StartAnalysis", "\n_______________________________________________________________________ \
1803 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1804 \n_______________________________________________________________________",
1805 fJDLName.Data(), cjobId);
1806 jobID = cjobId;
1807 }
1808 delete res;
1809 }
1810 } else {
d2a409b2 1811 // Submit for a range of enumeration of runs.
1812 Submit();
c57f56b7 1813 }
a8739e8a 1814
c57f56b7 1815 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1816 \n You may exit at any time and terminate the job later using the option <terminate> \
1817 \n ##################################################################################", jobID.Data());
bb885a9e 1818 gSystem->Exec("aliensh");
5513444a 1819 return kTRUE;
c57f56b7 1820}
1821
d2a409b2 1822//______________________________________________________________________________
1823void AliAnalysisAlien::Submit()
1824{
1825// Submit all master jobs.
1826 Int_t nmasterjobs = fInputFiles->GetEntries();
1827 Long_t tshoot = gSystem->Now();
1828 if (!fNsubmitted) SubmitNext();
1829 while (fNsubmitted < nmasterjobs) {
1830 Long_t now = gSystem->Now();
1831 if ((now-tshoot)>30000) {
1832 tshoot = now;
1833 SubmitNext();
1834 }
1835 }
1836}
1837
0f389141 1838//______________________________________________________________________________
1839void AliAnalysisAlien::SubmitMerging()
1840{
1841// Submit all merging jobs.
1842 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1843 gGrid->Cd(fGridOutputDir);
1844 TString mergeJDLName = fExecutable;
1845 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1846 Int_t ntosubmit = fInputFiles->GetEntries();
1847 printf("### Submitting %d merging jobs...\n", ntosubmit);
1848 for (Int_t i=0; i<ntosubmit; i++) {
1849 TString query;
f866cba5 1850 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
1851 runOutDir.ReplaceAll(".xml", "");
1852 if (fOutputToRunNo)
1853 query = Form("submit %s %s", mergeJDLName.Data(), runOutDir.Data());
1854 else
1855 query = Form("submit %s %03d", mergeJDLName.Data(), i);
0f389141 1856 printf("********* %s\n",query.Data());
1857 TGridResult *res = gGrid->Command(query);
1858 if (res) {
1859 const char *cjobId = res->GetKey(0,"jobId");
1860 if (!cjobId) {
1861 Error("StartAnalysis", "Your JDL %s could not be submitted", mergeJDLName.Data());
1862 return;
1863 } else {
1864 Info("StartAnalysis", "\n_______________________________________________________________________ \
1865 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1866 \n_______________________________________________________________________",
1867 mergeJDLName.Data(), cjobId);
1868 }
1869 delete res;
1870 }
1871 }
1872 if (!ntosubmit) return;
1873 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR MERGING JOBS HAVE FINISHED. #### \
1874 \n You may exit at any time and terminate the job later using the option <terminate> but disabling SetMergeViaJDL\
1875 \n ##################################################################################");
1876 gSystem->Exec("aliensh");
1877}
1878
d2a409b2 1879//______________________________________________________________________________
1880void AliAnalysisAlien::SubmitNext()
1881{
1882// Submit next bunch of master jobs if the queue is free.
1883 static Bool_t iscalled = kFALSE;
1884 static Int_t firstmaster = 0;
1885 static Int_t lastmaster = 0;
1886 static Int_t npermaster = 0;
1887 if (iscalled) return;
1888 iscalled = kTRUE;
1889 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1890 Int_t ntosubmit = 0;
1891 TGridResult *res;
1892 TString jobID = "";
1893 if (!fNsubmitted) ntosubmit = 1;
1894 else {
1895 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1896 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1897 // If last master not split, just return
1898 if (status != "SPLIT") {iscalled = kFALSE; return;}
1899 // No more than 100 waiting jobs
1900 if (nwaiting>100) {iscalled = kFALSE; return;}
1901 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1902 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1903 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1904 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1905 }
1906 Int_t nmasterjobs = fInputFiles->GetEntries();
1907 for (Int_t i=0; i<ntosubmit; i++) {
1908 // Submit for a range of enumeration of runs.
1909 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1910 TString query;
cd11251e 1911 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
1912 runOutDir.ReplaceAll(".xml", "");
1913 if (fOutputToRunNo)
1914 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
1915 else
1916 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1917 printf("********* %s\n",query.Data());
d2a409b2 1918 res = gGrid->Command(query);
1919 if (res) {
98ca124f 1920 TString cjobId1 = res->GetKey(0,"jobId");
1921 if (!cjobId1.Length()) {
d2a409b2 1922 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1923 iscalled = kFALSE;
1924 return;
1925 } else {
1926 Info("StartAnalysis", "\n_______________________________________________________________________ \
1927 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1928 \n_______________________________________________________________________",
98ca124f 1929 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1930 jobID += cjobId1;
1931 jobID += " ";
98ca124f 1932 lastmaster = cjobId1.Atoi();
d2a409b2 1933 if (!firstmaster) firstmaster = lastmaster;
1934 fNsubmitted++;
1935 }
1936 delete res;
1937 }
1938 }
1939 iscalled = kFALSE;
1940}
1941
c57f56b7 1942//______________________________________________________________________________
1943void AliAnalysisAlien::WriteAnalysisFile()
1944{
f10e8481 1945// Write current analysis manager into the file <analysisFile>
1946 TString analysisFile = fExecutable;
1947 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1948 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1949 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1950 if (!mgr || !mgr->IsInitialized()) {
1951 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1952 return;
1953 }
1954 // Check analysis type
1955 TObject *handler;
1956 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1957 handler = (TObject*)mgr->GetInputEventHandler();
1958 if (handler) {
1959 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1960 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1961 }
1962 TDirectory *cdir = gDirectory;
f10e8481 1963 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1964 if (file) {
fe2d7fc2 1965 // Skip task Terminate calls for the grid job
1966 mgr->SetSkipTerminate(kTRUE);
1967 // Unless merging makes no sense
1968 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 1969 mgr->Write();
1970 delete file;
fe2d7fc2 1971 // Enable termination for local jobs
1972 mgr->SetSkipTerminate(kFALSE);
c57f56b7 1973 }
1974 if (cdir) cdir->cd();
f10e8481 1975 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1976 }
1977 Bool_t copy = kTRUE;
1978 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1979 if (copy) {
1980 CdWork();
1981 TString workdir = gGrid->GetHomeDirectory();
1982 workdir += fGridWorkingDir;
f10e8481 1983 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1984 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1985 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1986 }
1987}
1988
1989//______________________________________________________________________________
1990void AliAnalysisAlien::WriteAnalysisMacro()
1991{
1992// Write the analysis macro that will steer the analysis in grid mode.
1993 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1994 ofstream out;
1995 out.open(fAnalysisMacro.Data(), ios::out);
1996 if (!out.good()) {
1997 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1998 return;
1999 }
5b273635 2000 Bool_t hasSTEERBase = kFALSE;
2001 Bool_t hasESD = kFALSE;
2002 Bool_t hasAOD = kFALSE;
2003 Bool_t hasANALYSIS = kFALSE;
2004 Bool_t hasANALYSISalice = kFALSE;
2005 Bool_t hasCORRFW = kFALSE;
c57f56b7 2006 TString func = fAnalysisMacro;
2007 TString type = "ESD";
2008 TString comment = "// Analysis using ";
2009 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
2010 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
2011 type = "AOD";
2012 comment += "AOD";
2013 }
0df6ccf2 2014 if (type!="AOD" && fFriendChainName!="") {
2015 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
2016 return;
2017 }
c57f56b7 2018 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
2019 else comment += " data";
2020 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
2021 func.ReplaceAll(".C", "");
2022 out << "void " << func.Data() << "()" << endl;
2023 out << "{" << endl;
2024 out << comment.Data() << endl;
2025 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 2026 out << " TStopwatch timer;" << endl;
2027 out << " timer.Start();" << endl << endl;
c57f56b7 2028 out << "// load base root libraries" << endl;
2029 out << " gSystem->Load(\"libTree\");" << endl;
2030 out << " gSystem->Load(\"libGeom\");" << endl;
2031 out << " gSystem->Load(\"libVMC\");" << endl;
2032 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 2033 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 2034 if (fAdditionalRootLibs.Length()) {
47a4137d 2035 // in principle libtree /lib geom libvmc etc. can go into this list, too
2036 out << "// Add aditional libraries" << endl;
2037 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2038 TIter next(list);
2039 TObjString *str;
2040 while((str=(TObjString*)next())) {
2041 if (str->GetString().Contains(".so"))
2042 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
d5c6455a 2043 }
47a4137d 2044 if (list) delete list;
d5c6455a 2045 }
47a4137d 2046 out << "// include path" << endl;
2047 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2048 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
57377eb5 2049 out << "// Load analysis framework libraries" << endl;
4e5c5506 2050 if (!fPackages) {
4e5c5506 2051 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2052 out << " gSystem->Load(\"libESD\");" << endl;
2053 out << " gSystem->Load(\"libAOD\");" << endl;
2054 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 2055 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2056 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 2057 } else {
4e5c5506 2058 TIter next(fPackages);
2059 TObject *obj;
57377eb5 2060 TString pkgname;
5b273635 2061 TString setupPar = "AliAnalysisAlien::SetupPar";
57377eb5 2062 while ((obj=next())) {
2063 pkgname = obj->GetName();
4478e6f1 2064 if (pkgname == "STEERBase" ||
2065 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2066 if (pkgname == "ESD" ||
2067 pkgname == "ESD.par") hasESD = kTRUE;
2068 if (pkgname == "AOD" ||
2069 pkgname == "AOD.par") hasAOD = kTRUE;
2070 if (pkgname == "ANALYSIS" ||
2071 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2072 if (pkgname == "ANALYSISalice" ||
2073 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2074 if (pkgname == "CORRFW" ||
2075 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
5b273635 2076 }
2077 if (hasANALYSISalice) setupPar = "SetupPar";
57377eb5 2078 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2079 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
57377eb5 2080 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2081 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
57377eb5 2082 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2083 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
57377eb5 2084 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2085 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
57377eb5 2086 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2087 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
57377eb5 2088 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2089 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
57377eb5 2090 out << "// Compile other par packages" << endl;
2091 next.Reset();
fcc9bb6f 2092 while ((obj=next())) {
2093 pkgname = obj->GetName();
4478e6f1 2094 if (pkgname == "STEERBase" ||
2095 pkgname == "STEERBase.par" ||
2096 pkgname == "ESD" ||
2097 pkgname == "ESD.par" ||
2098 pkgname == "AOD" ||
2099 pkgname == "AOD.par" ||
2100 pkgname == "ANALYSIS" ||
2101 pkgname == "ANALYSIS.par" ||
2102 pkgname == "ANALYSISalice" ||
2103 pkgname == "ANALYSISalice.par" ||
2104 pkgname == "CORRFW" ||
2105 pkgname == "CORRFW.par") continue;
5b273635 2106 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 2107 }
4e5c5506 2108 }
6da75e0b 2109 if (fAdditionalLibs.Length()) {
2110 out << "// Add aditional AliRoot libraries" << endl;
2111 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2112 TIter next(list);
2113 TObjString *str;
2114 while((str=(TObjString*)next())) {
2115 if (str->GetString().Contains(".so"))
2116 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2117 }
2118 if (list) delete list;
2119 }
2120 out << endl;
c57f56b7 2121 out << "// analysis source to be compiled at runtime (if any)" << endl;
2122 if (fAnalysisSource.Length()) {
2123 TObjArray *list = fAnalysisSource.Tokenize(" ");
2124 TIter next(list);
2125 TObjString *str;
2126 while((str=(TObjString*)next())) {
2127 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2128 }
2129 if (list) delete list;
2130 }
2131 out << endl;
0f389141 2132 if (fFastReadOption) {
2133 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. Note that this may skip some files that could be accessed !!!");
2134 out << "// fast xrootd reading enabled" << endl;
2135 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
2136 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",5);" << endl;
2137 out << " gEnv->SetValue(\"XNet.RequestTimeout\",5);" << endl;
2138 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
2139 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",5);" << endl;
2140 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2141 }
c57f56b7 2142 out << "// connect to AliEn and make the chain" << endl;
2143 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2144 if (IsUsingTags()) {
2145 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
2146 } else {
76535c98 2147 if(fFriendChainName!="AliAOD.VertexingHF.root") {
2148 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
2149 } else {
2150 out << " // Check if the macro to create the chain was provided" << endl;
2151 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
2152 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
2153 out << " return;" << endl;
2154 out << " }" << endl;
2155 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
2156 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
2157 }
c57f56b7 2158 }
2159 out << "// read the analysis manager from file" << endl;
f10e8481 2160 TString analysisFile = fExecutable;
2161 analysisFile.ReplaceAll(".sh", ".root");
2162 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 2163 out << " if (!file) return;" << endl;
2164 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2165 out << " AliAnalysisManager *mgr = 0;" << endl;
2166 out << " TKey *key;" << endl;
2167 out << " while ((key=(TKey*)nextkey())) {" << endl;
2168 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2169 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2170 out << " };" << endl;
2171 out << " if (!mgr) {" << endl;
f10e8481 2172 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 2173 out << " return;" << endl;
2174 out << " }" << endl << endl;
2175 out << " mgr->PrintStatus();" << endl;
52b6a92b 2176 if (AliAnalysisManager::GetAnalysisManager()) {
f866cba5 2177 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
52b6a92b 2178 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
de52b69c 2179 } else {
2180 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
52b6a92b 2181 }
2182 }
c57f56b7 2183 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 2184 out << " timer.Stop();" << endl;
2185 out << " timer.Print();" << endl;
c57f56b7 2186 out << "}" << endl << endl;
2187 if (IsUsingTags()) {
2188 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
2189 out << "{" << endl;
2190 out << "// Create a chain using tags from the xml file." << endl;
2191 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
2192 out << " if (!coll) {" << endl;
2193 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2194 out << " return NULL;" << endl;
2195 out << " }" << endl;
2196 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
2197 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
2198 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
2199 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
2200 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
2201 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
2202 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
2203 out << " // Check if the cuts configuration file was provided" << endl;
2204 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
2205 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
2206 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2207 out << " }" << endl;
0df6ccf2 2208 if (fFriendChainName=="") {
2209 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
2210 } else {
2211 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
2212 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
2213 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
2214 }
c57f56b7 2215 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
2216 out << " chain->ls();" << endl;
2217 out << " return chain;" << endl;
fcc9bb6f 2218 out << "}" << endl << endl;
c57f56b7 2219 if (gSystem->AccessPathName("ConfigureCuts.C")) {
2220 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
2221 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
2222 msg += " AliLHCTagCuts *lhcCuts,\n";
2223 msg += " AliDetectorTagCuts *detCuts,\n";
2224 msg += " AliEventTagCuts *evCuts)";
2225 Info("WriteAnalysisMacro", msg.Data());
2226 }
0df6ccf2 2227 }
2228 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 2229 out <<"//________________________________________________________________________________" << endl;
c57f56b7 2230 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
2231 out << "{" << endl;
2232 out << "// Create a chain using url's from xml file" << endl;
2233 out << " TString treename = type;" << endl;
2234 out << " treename.ToLower();" << endl;
2235 out << " treename += \"Tree\";" << endl;
e02fee64 2236 out << " printf(\"***************************************\\n\");" << endl;
2237 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
2238 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 2239 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
2240 out << " if (!coll) {" << endl;
2241 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
2242 out << " return NULL;" << endl;
2243 out << " }" << endl;
2244 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 2245 if(fFriendChainName!="") {
2246 out << " TChain *chainFriend = new TChain(treename);" << endl;
2247 }
c57f56b7 2248 out << " coll->Reset();" << endl;
0df6ccf2 2249 out << " while (coll->Next()) {" << endl;
2250 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
2251 if(fFriendChainName!="") {
2252 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
2253 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2254 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2255 out << " chainFriend->Add(fileFriend.Data());" << endl;
2256 }
2257 out << " }" << endl;
c57f56b7 2258 out << " if (!chain->GetNtrees()) {" << endl;
2259 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
2260 out << " return NULL;" << endl;
2261 out << " }" << endl;
0df6ccf2 2262 if(fFriendChainName!="") {
2263 out << " chain->AddFriend(chainFriend);" << endl;
2264 }
c57f56b7 2265 out << " return chain;" << endl;
fcc9bb6f 2266 out << "}" << endl << endl;
c57f56b7 2267 }
5b273635 2268 if (hasANALYSISalice) {
2269 out <<"//________________________________________________________________________________" << endl;
2270 out << "Bool_t SetupPar(const char *package) {" << endl;
2271 out << "// Compile the package and set it up." << endl;
2272 out << " TString pkgdir = package;" << endl;
2273 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2274 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2275 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2276 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2277 out << " // Check for BUILD.sh and execute" << endl;
2278 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2279 out << " printf(\"*******************************\\n\");" << endl;
2280 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2281 out << " printf(\"*******************************\\n\");" << endl;
2282 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2283 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2284 out << " gSystem->ChangeDirectory(cdir);" << endl;
2285 out << " return kFALSE;" << endl;
2286 out << " }" << endl;
2287 out << " } else {" << endl;
2288 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2289 out << " gSystem->ChangeDirectory(cdir);" << endl;
2290 out << " return kFALSE;" << endl;
2291 out << " }" << endl;
2292 out << " // Check for SETUP.C and execute" << endl;
2293 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2294 out << " printf(\"*******************************\\n\");" << endl;
2295 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2296 out << " printf(\"*******************************\\n\");" << endl;
2297 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2298 out << " } else {" << endl;
2299 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2300 out << " gSystem->ChangeDirectory(cdir);" << endl;
2301 out << " return kFALSE;" << endl;
2302 out << " }" << endl;
2303 out << " // Restore original workdir" << endl;
2304 out << " gSystem->ChangeDirectory(cdir);" << endl;
2305 out << " return kTRUE;" << endl;
2306 out << "}" << endl;
2307 }
c57f56b7 2308 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2309 }
2310 Bool_t copy = kTRUE;
2311 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2312 if (copy) {
2313 CdWork();
2314 TString workdir = gGrid->GetHomeDirectory();
2315 workdir += fGridWorkingDir;
2316 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2317 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2318 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2319 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2320 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2321 }
2322 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2323 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2324 }
2325}
2326
0f389141 2327//______________________________________________________________________________
2328void AliAnalysisAlien::WriteMergingMacro()
2329{
2330// Write a macro to merge the outputs per master job.
2331 if (!fMergeViaJDL) return;
2332 if (!fOutputFiles.Length()) {
2333 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2334 return;
2335 }
2336 TString mergingMacro = fExecutable;
2337 mergingMacro.ReplaceAll(".sh","_merge.C");
2338 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2339 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2340 ofstream out;
2341 out.open(mergingMacro.Data(), ios::out);
2342 if (!out.good()) {
2343 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
2344 return;
2345 }
5b273635 2346 Bool_t hasSTEERBase = kFALSE;
2347 Bool_t hasESD = kFALSE;
2348 Bool_t hasAOD = kFALSE;
2349 Bool_t hasANALYSIS = kFALSE;
2350 Bool_t hasANALYSISalice = kFALSE;
2351 Bool_t hasCORRFW = kFALSE;
0f389141 2352 TString func = mergingMacro;
2353 TString comment;
2354 func.ReplaceAll(".C", "");
2355 out << "void " << func.Data() << "(const char *dir)" << endl;
2356 out << "{" << endl;
2357 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
2358 out << " TStopwatch timer;" << endl;
2359 out << " timer.Start();" << endl << endl;
2360 out << "// load base root libraries" << endl;
2361 out << " gSystem->Load(\"libTree\");" << endl;
2362 out << " gSystem->Load(\"libGeom\");" << endl;
2363 out << " gSystem->Load(\"libVMC\");" << endl;
2364 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
2365 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
2366 if (fAdditionalRootLibs.Length()) {
2367 // in principle libtree /lib geom libvmc etc. can go into this list, too
2368 out << "// Add aditional libraries" << endl;
2369 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
2370 TIter next(list);
2371 TObjString *str;
2372 while((str=(TObjString*)next())) {
2373 if (str->GetString().Contains(".so"))
2374 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2375 }
2376 if (list) delete list;
2377 }
2378 out << "// include path" << endl;
2379 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
2380 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
2381 out << "// Load analysis framework libraries" << endl;
2382 if (!fPackages) {
2383 out << " gSystem->Load(\"libSTEERBase\");" << endl;
2384 out << " gSystem->Load(\"libESD\");" << endl;
2385 out << " gSystem->Load(\"libAOD\");" << endl;
2386 out << " gSystem->Load(\"libANALYSIS\");" << endl;
2387 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
2388 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
2389 } else {
2390 TIter next(fPackages);
2391 TObject *obj;
2392 TString pkgname;
5b273635 2393 TString setupPar = "AliAnalysisAlien::SetupPar";
0f389141 2394 while ((obj=next())) {
2395 pkgname = obj->GetName();
2396 if (pkgname == "STEERBase" ||
2397 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
2398 if (pkgname == "ESD" ||
2399 pkgname == "ESD.par") hasESD = kTRUE;
2400 if (pkgname == "AOD" ||
2401 pkgname == "AOD.par") hasAOD = kTRUE;
2402 if (pkgname == "ANALYSIS" ||
2403 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
2404 if (pkgname == "ANALYSISalice" ||
2405 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
2406 if (pkgname == "CORRFW" ||
2407 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
2408 }
5b273635 2409 if (hasANALYSISalice) setupPar = "SetupPar";
0f389141 2410 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
5b273635 2411 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
0f389141 2412 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
5b273635 2413 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
0f389141 2414 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
5b273635 2415 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
0f389141 2416 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
5b273635 2417 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
0f389141 2418 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
5b273635 2419 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
0f389141 2420 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
5b273635 2421 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
0f389141 2422 out << "// Compile other par packages" << endl;
2423 next.Reset();
2424 while ((obj=next())) {
2425 pkgname = obj->GetName();
2426 if (pkgname == "STEERBase" ||
2427 pkgname == "STEERBase.par" ||
2428 pkgname == "ESD" ||
2429 pkgname == "ESD.par" ||
2430 pkgname == "AOD" ||
2431 pkgname == "AOD.par" ||
2432 pkgname == "ANALYSIS" ||
2433 pkgname == "ANALYSIS.par" ||
2434 pkgname == "ANALYSISalice" ||
2435 pkgname == "ANALYSISalice.par" ||
2436 pkgname == "CORRFW" ||
2437 pkgname == "CORRFW.par") continue;
5b273635 2438 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
0f389141 2439 }
2440 }
2441 if (fAdditionalLibs.Length()) {
2442 out << "// Add aditional AliRoot libraries" << endl;
2443 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2444 TIter next(list);
2445 TObjString *str;
2446 while((str=(TObjString*)next())) {
2447 if (str->GetString().Contains(".so"))
2448 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
2449 }
2450 if (list) delete list;
2451 }
2452 out << endl;
2453 out << "// Analysis source to be compiled at runtime (if any)" << endl;
2454 if (fAnalysisSource.Length()) {
2455 TObjArray *list = fAnalysisSource.Tokenize(" ");
2456 TIter next(list);
2457 TObjString *str;
2458 while((str=(TObjString*)next())) {
2459 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
2460 }
2461 if (list) delete list;
2462 }
2463 out << endl;
2464 if (fFastReadOption) {
2465 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
2466 out << "// fast xrootd reading enabled" << endl;
2467 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
2468 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",5);" << endl;
2469 out << " gEnv->SetValue(\"XNet.RequestTimeout\",5);" << endl;
2470 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
2471 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",5);" << endl;
2472 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
2473 }
2474 out << "// Connect to AliEn" << endl;
2475 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
2476 out << " TString outputDir = \"" << fGridOutputDir << "/\";" << endl;
2477 out << " outputDir += dir;" << endl;
2478 out << " TString outputFiles = \"" << fOutputFiles << "\";" << endl;
2479 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
2480 out << " TObjArray *list = outputFiles.Tokenize(\" \");" << endl;
2481 out << " TIter *iter = new TIter(list);" << endl;
2482 out << " TObjString *str;" << endl;
2483 out << " TString output_file;" << endl;
2484 out << " Bool_t merged = kTRUE;" << endl;
2485 out << " while((str=(TObjString*)iter->Next())) {" << endl;
2486 out << " output_file = str->GetString();" << endl;
2487 out << " Int_t index = output_file.Index(\"@\");" << endl;
2488 out << " if (index > 0) output_file.Remove(index);" << endl;
2489 out << " // Skip already merged outputs" << endl;
2490 out << " if (!gSystem->AccessPathName(output_file)) {" << endl;
2491 out << " printf(\"Output file <%s> found. Not merging again.\",output_file.Data());" << endl;
2492 out << " continue;" << endl;
2493 out << " }" << endl;
2494 out << " if (mergeExcludes.Contains(output_file.Data())) continue;" << endl;
2495 out << " merged = AliAnalysisAlien::MergeOutput(output_file, outputDir, " << fMaxMergeFiles << ");" << endl;
2496 out << " if (!merged) {" << endl;
2497 out << " printf(\"ERROR: Cannot merge %s\\n\", output_file.Data());" << endl;
2498 out << " return;" << endl;
2499 out << " }" << endl;
2500 out << " }" << endl;
f866cba5 2501 out << "// read the analysis manager from file" << endl;
2502 TString analysisFile = fExecutable;
2503 analysisFile.ReplaceAll(".sh", ".root");
2504 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
2505 out << " if (!file) return;" << endl;
2506 out << " TIter nextkey(file->GetListOfKeys());" << endl;
2507 out << " AliAnalysisManager *mgr = 0;" << endl;
2508 out << " TKey *key;" << endl;
2509 out << " while ((key=(TKey*)nextkey())) {" << endl;
2510 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
2511 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
2512 out << " };" << endl;
2513 out << " if (!mgr) {" << endl;
2514 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
2515 out << " return;" << endl;
2516 out << " }" << endl << endl;
b385fec0 2517 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
f866cba5 2518 out << " mgr->PrintStatus();" << endl;
2519 if (AliAnalysisManager::GetAnalysisManager()) {
2520 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
2521 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
2522 } else {
2523 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
2524 }
2525 }
2526 out << " mgr->StartAnalysis(\"gridterminate\");" << endl;
0f389141 2527 out << "}" << endl << endl;
5b273635 2528 if (hasANALYSISalice) {
2529 out <<"//________________________________________________________________________________" << endl;
2530 out << "Bool_t SetupPar(const char *package) {" << endl;
2531 out << "// Compile the package and set it up." << endl;
2532 out << " TString pkgdir = package;" << endl;
2533 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2534 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2535 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2536 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2537 out << " // Check for BUILD.sh and execute" << endl;
2538 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2539 out << " printf(\"*******************************\\n\");" << endl;
2540 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2541 out << " printf(\"*******************************\\n\");" << endl;
2542 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2543 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2544 out << " gSystem->ChangeDirectory(cdir);" << endl;
2545 out << " return kFALSE;" << endl;
2546 out << " }" << endl;
2547 out << " } else {" << endl;
2548 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2549 out << " gSystem->ChangeDirectory(cdir);" << endl;
2550 out << " return kFALSE;" << endl;
2551 out << " }" << endl;
2552 out << " // Check for SETUP.C and execute" << endl;
2553 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2554 out << " printf(\"*******************************\\n\");" << endl;
2555 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2556 out << " printf(\"*******************************\\n\");" << endl;
2557 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2558 out << " } else {" << endl;
2559 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2560 out << " gSystem->ChangeDirectory(cdir);" << endl;
2561 out << " return kFALSE;" << endl;
2562 out << " }" << endl;
2563 out << " // Restore original workdir" << endl;
2564 out << " gSystem->ChangeDirectory(cdir);" << endl;
2565 out << " return kTRUE;" << endl;
2566 out << "}" << endl;
2567 }
0f389141 2568 }
2569 Bool_t copy = kTRUE;
2570 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2571 if (copy) {
2572 CdWork();
2573 TString workdir = gGrid->GetHomeDirectory();
2574 workdir += fGridWorkingDir;
2575 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
2576 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
2577 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
2578 }
2579}
2580
2581//______________________________________________________________________________
2582Bool_t AliAnalysisAlien::SetupPar(const char *package)
2583{
2584// Compile the par file archive pointed by <package>. This must be present in the current durectory.
2585// Note that for loading the compiled library. The current directory should have precedence in
2586// LD_LIBRARY_PATH
2587 TString pkgdir = package;
2588 pkgdir.ReplaceAll(".par","");
2589 gSystem->Exec(Form("tar xvzf %s.par", pkgdir.Data()));
2590 TString cdir = gSystem->WorkingDirectory();
2591 gSystem->ChangeDirectory(pkgdir);
2592 // Check for BUILD.sh and execute
2593 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
2594 printf("**************************************************\n");
2595 printf("*** Building PAR archive %s\n", package);
2596 printf("**************************************************\n");
2597 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
2598 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
2599 gSystem->ChangeDirectory(cdir);
2600 return kFALSE;
2601 }
2602 } else {
2603 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
2604 gSystem->ChangeDirectory(cdir);
2605 return kFALSE;
2606 }
2607 // Check for SETUP.C and execute
2608 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
2609 printf("**************************************************\n");
2610 printf("*** Setup PAR archive %s\n", package);
2611 printf("**************************************************\n");
2612 gROOT->Macro("PROOF-INF/SETUP.C");
2613 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
2614 } else {
2615 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
2616 gSystem->ChangeDirectory(cdir);
2617 return kFALSE;
2618 }
2619 // Restore original workdir
2620 gSystem->ChangeDirectory(cdir);
2621 return kTRUE;
2622}
2623
c57f56b7 2624//______________________________________________________________________________
2625void AliAnalysisAlien::WriteExecutable()
2626{
2627// Generate the alien executable script.
2628 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2629 ofstream out;
2630 out.open(fExecutable.Data(), ios::out);
2631 if (out.bad()) {
5513444a 2632 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2633 return;
2634 }
2635 out << "#!/bin/bash" << endl;
c57f56b7 2636 out << "echo \"=========================================\"" << endl;
2637 out << "echo \"############## PATH : ##############\"" << endl;
2638 out << "echo $PATH" << endl;
2639 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2640 out << "echo $LD_LIBRARY_PATH" << endl;
2641 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2642 out << "echo $ROOTSYS" << endl;
2643 out << "echo \"############## which root : ##############\"" << endl;
2644 out << "which root" << endl;
2645 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2646 out << "echo $ALICE_ROOT" << endl;
2647 out << "echo \"############## which aliroot : ##############\"" << endl;
2648 out << "which aliroot" << endl;
9c5ddadc 2649 out << "echo \"############## system limits : ##############\"" << endl;
2650 out << "ulimit -a" << endl;
2651 out << "echo \"############## memory : ##############\"" << endl;
2652 out << "free -m" << endl;
c57f56b7 2653 out << "echo \"=========================================\"" << endl << endl;
74a53467 2654 // Make sure we can properly compile par files
2655 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2656 out << fExecutableCommand << " ";
631c0b05 2657 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2658 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2659 out << "echo \"############## memory after: ##############\"" << endl;
2660 out << "free -m" << endl;
2661 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2662 out << "dmesg | tail -n 10" << endl;
c57f56b7 2663 }
2664 Bool_t copy = kTRUE;
2665 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2666 if (copy) {
2667 CdWork();
2668 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2669 TString bindir = Form("%s/bin", workdir.Data());
2670 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2671 workdir += fGridWorkingDir;
2672 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2673 if (FileExists(executable)) gGrid->Rm(executable);
2674 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2675 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2676 }
2677}
2678
0f389141 2679//______________________________________________________________________________
2680void AliAnalysisAlien::WriteMergeExecutable()
2681{
2682// Generate the alien executable script for the merging job.
2683 if (!fMergeViaJDL) return;
2684 TString mergeExec = fExecutable;
2685 mergeExec.ReplaceAll(".sh", "_merge.sh");
2686 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2687 ofstream out;
2688 out.open(mergeExec.Data(), ios::out);
2689 if (out.bad()) {
2690 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
2691 return;
2692 }
2693 out << "#!/bin/bash" << endl;
2694 out << "echo \"=========================================\"" << endl;
2695 out << "echo \"############## PATH : ##############\"" << endl;
2696 out << "echo $PATH" << endl;
2697 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2698 out << "echo $LD_LIBRARY_PATH" << endl;
2699 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2700 out << "echo $ROOTSYS" << endl;
2701 out << "echo \"############## which root : ##############\"" << endl;
2702 out << "which root" << endl;
2703 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2704 out << "echo $ALICE_ROOT" << endl;
2705 out << "echo \"############## which aliroot : ##############\"" << endl;
2706 out << "which aliroot" << endl;
2707 out << "echo \"############## system limits : ##############\"" << endl;
2708 out << "ulimit -a" << endl;
2709 out << "echo \"############## memory : ##############\"" << endl;
2710 out << "free -m" << endl;
2711 out << "echo \"=========================================\"" << endl << endl;
2712 // Make sure we can properly compile par files
2713 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
2714 TString mergeMacro = fExecutable;
2715 mergeMacro.ReplaceAll(".sh", "_merge.C");
2716 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
2717 out << fExecutableCommand << " " << "$ARG" << endl;
2718 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
2719 out << "echo \"############## memory after: ##############\"" << endl;
2720 out << "free -m" << endl;
2721 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2722 out << "dmesg | tail -n 10" << endl;
2723 }
2724 Bool_t copy = kTRUE;
2725 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2726 if (copy) {
2727 CdWork();
2728 TString workdir = gGrid->GetHomeDirectory();
2729 TString bindir = Form("%s/bin", workdir.Data());
2730 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
2731 workdir += fGridWorkingDir;
2732 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
2733 if (FileExists(executable)) gGrid->Rm(executable);
2734 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
2735 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
2736 }
2737}
2738
c57f56b7 2739//______________________________________________________________________________
5513444a 2740void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2741{
2742// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2743// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2744// Next lines: full_path_to_dataset XXX (XXX is a string)
2745// To submit, one has to: submit jdl XXX for all lines
2746 ofstream out;
2747 out.open(filename, ios::out);
2748 if (out.bad()) {
2749 Error("WriteProductionFile", "Bad file name: %s", filename);
2750 return;
2751 }
2752 TString workdir = gGrid->GetHomeDirectory();
2753 workdir += fGridWorkingDir;
f5e8c702 2754 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2755 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2756 out << locjdl << " " << njobspermaster << endl;
5513444a 2757 Int_t nmasterjobs = fInputFiles->GetEntries();
2758 for (Int_t i=0; i<nmasterjobs; i++) {
409b4ada 2759 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2760 runOutDir.ReplaceAll(".xml", "");
2761 if (fOutputToRunNo)
2762 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
2763 else
2764 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2765 }
2766 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
fdbbc7be 2767 if (FileExists(filename)) gGrid->Rm(filename);
5513444a 2768 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2769}
2770
2771//______________________________________________________________________________
0f389141 2772void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
c57f56b7 2773{
2774// Generate the alien validation script.
2775 // Generate the validation script
2776 TObjString *os;
0d5d317c 2777 TString validationScript = fExecutable;
0f389141 2778 if (merge) validationScript.ReplaceAll(".sh", "_mergevalidation.sh");
2779 else validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2780 if (!Connect()) {
2781 Error("WriteValidationScript", "Alien connection required");
2782 return;
2783 }
2784 TString out_stream = "";
2785 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2786 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2787 ofstream out;
0d5d317c 2788 out.open(validationScript, ios::out);
c57f56b7 2789 out << "#!/bin/bash" << endl;
2790 out << "##################################################" << endl;
2791 out << "validateout=`dirname $0`" << endl;
2792 out << "validatetime=`date`" << endl;
2793 out << "validated=\"0\";" << endl;
2794 out << "error=0" << endl;
2795 out << "if [ -z $validateout ]" << endl;
2796 out << "then" << endl;
2797 out << " validateout=\".\"" << endl;
2798 out << "fi" << endl << endl;
2799 out << "cd $validateout;" << endl;
2800 out << "validateworkdir=`pwd`;" << endl << endl;
2801 out << "echo \"*******************************************************\"" << out_stream << endl;
2802 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2803 out << "" << endl;
2804 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2805 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2806 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2807 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2808 out << "ls -la ./" << out_stream << endl;
2809 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2810 out << "##################################################" << endl;
ebec370a 2811
2812 out << "" << endl;
2813 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2814 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2815 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2816 out << "" << endl;
2817
2818 out << "if [ ! -f stderr ] ; then" << endl;
2819 out << " error=1" << endl;
2820 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2821 out << " echo \"Error = $error\" " << out_stream << endl;
2822 out << "fi" << endl;
2823
2824 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2825 out << " error=1" << endl;
2826 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2827 out << " echo \"$parArch\" " << out_stream << endl;
2828 out << " echo \"Error = $error\" " << out_stream << endl;
2829 out << "fi" << endl;
2830
2831 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2832 out << " error=1" << endl;
2833 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2834 out << " echo \"$segViol\" " << out_stream << endl;
2835 out << " echo \"Error = $error\" " << out_stream << endl;
2836 out << "fi" << endl;
2837
2838 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2839 out << " error=1" << endl;
2840 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2841 out << " echo \"$segFault\" " << out_stream << endl;
2842 out << " echo \"Error = $error\" " << out_stream << endl;
2843 out << "fi" << endl;
2844
2845 // Part dedicated to the specific analyses running into the train
2846
c57f56b7 2847 TObjArray *arr = fOutputFiles.Tokenize(" ");
2848 TIter next1(arr);
2849 TString output_file;
2850 while ((os=(TObjString*)next1())) {
2851 output_file = os->GetString();
2852 Int_t index = output_file.Index("@");
2853 if (index > 0) output_file.Remove(index);
0f389141 2854 if (merge && fMergeExcludes.Contains(output_file)) continue;
c57f56b7 2855 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2856 out << " error=1" << endl;
2857 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2858 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2859 out << "fi" << endl;
2860 }
2861 delete arr;
f866cba5 2862 if (!merge) {
2863 out << "if ! [ -f outputs_valid ] ; then" << endl;
2864 out << " error=1" << endl;
2865 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2866 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2867 out << "fi" << endl;
2868 }
923e2ca5 2869
c57f56b7 2870 out << "if [ $error = 0 ] ; then" << endl;
2871 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2872 out << "fi" << endl;
2873
2874 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2875 out << "echo \"*******************************************************\"" << out_stream << endl;
2876 out << "cd -" << endl;
2877 out << "exit $error" << endl;
2878 }
2879 Bool_t copy = kTRUE;
2880 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2881 if (copy) {
2882 CdWork();
2883 TString workdir = gGrid->GetHomeDirectory();
2884 workdir += fGridWorkingDir;
0d5d317c 2885 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2886 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2887 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2888 }
2889}