]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Read histos from common output file (Chiara B)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
cd11251e 57 fOutputToRunNo(0),
c57f56b7 58 fRunNumbers(),
59 fExecutable(),
0a1c1f7f 60 fExecutableCommand(),
c57f56b7 61 fArguments(),
631c0b05 62 fExecutableArgs(),
c57f56b7 63 fAnalysisMacro(),
64 fAnalysisSource(),
65 fAdditionalLibs(),
66 fSplitMode(),
67 fAPIVersion(),
68 fROOTVersion(),
69 fAliROOTVersion(),
648174cf 70 fExternalPackages(),
c57f56b7 71 fUser(),
72 fGridWorkingDir(),
73 fGridDataDir(),
74 fDataPattern(),
75 fGridOutputDir(),
76 fOutputArchive(),
77 fOutputFiles(),
78 fInputFormat(),
e7c71df0 79 fDatasetName(),
c57f56b7 80 fJDLName(),
bb885a9e 81 fMergeExcludes(),
f965131e 82 fIncludePath(),
bb885a9e 83 fCloseSE(),
0df6ccf2 84 fFriendChainName(),
c6cb3634 85 fJobTag(),
648174cf 86 fOutputSingle(),
5fce53f4 87 fRunPrefix(),
4e5c5506 88 fInputFiles(0),
89 fPackages(0)
c57f56b7 90{
91// Dummy ctor.
92 SetDefaults();
93}
94
95//______________________________________________________________________________
96AliAnalysisAlien::AliAnalysisAlien(const char *name)
97 :AliAnalysisGrid(name),
98 fGridJDL(NULL),
99 fPrice(0),
100 fTTL(0),
101 fSplitMaxInputFileNumber(0),
102 fMaxInitFailed(0),
103 fMasterResubmitThreshold(0),
bb885a9e 104 fNtestFiles(0),
319593fb 105 fNrunsPerMaster(0),
16a4353c 106 fMaxMergeFiles(0),
d2a409b2 107 fNsubmitted(0),
a3e84053 108 fProductionMode(0),
cd11251e 109 fOutputToRunNo(0),
c57f56b7 110 fRunNumbers(),
111 fExecutable(),
0a1c1f7f 112 fExecutableCommand(),
c57f56b7 113 fArguments(),
631c0b05 114 fExecutableArgs(),
c57f56b7 115 fAnalysisMacro(),
116 fAnalysisSource(),
117 fAdditionalLibs(),
118 fSplitMode(),
119 fAPIVersion(),
120 fROOTVersion(),
121 fAliROOTVersion(),
648174cf 122 fExternalPackages(),
c57f56b7 123 fUser(),
124 fGridWorkingDir(),
125 fGridDataDir(),
126 fDataPattern(),
127 fGridOutputDir(),
128 fOutputArchive(),
129 fOutputFiles(),
130 fInputFormat(),
e7c71df0 131 fDatasetName(),
c57f56b7 132 fJDLName(),
bb885a9e 133 fMergeExcludes(),
f965131e 134 fIncludePath(),
bb885a9e 135 fCloseSE(),
0df6ccf2 136 fFriendChainName(),
c6cb3634 137 fJobTag(),
648174cf 138 fOutputSingle(),
5fce53f4 139 fRunPrefix(),
4e5c5506 140 fInputFiles(0),
141 fPackages(0)
c57f56b7 142{
143// Default ctor.
144 SetDefaults();
145}
146
147//______________________________________________________________________________
148AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
149 :AliAnalysisGrid(other),
150 fGridJDL(NULL),
151 fPrice(other.fPrice),
152 fTTL(other.fTTL),
153 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
154 fMaxInitFailed(other.fMaxInitFailed),
155 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 156 fNtestFiles(other.fNtestFiles),
319593fb 157 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 158 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 159 fNsubmitted(other.fNsubmitted),
a3e84053 160 fProductionMode(other.fProductionMode),
cd11251e 161 fOutputToRunNo(other.fOutputToRunNo),
c57f56b7 162 fRunNumbers(other.fRunNumbers),
163 fExecutable(other.fExecutable),
0a1c1f7f 164 fExecutableCommand(other.fExecutableCommand),
c57f56b7 165 fArguments(other.fArguments),
631c0b05 166 fExecutableArgs(other.fExecutableArgs),
c57f56b7 167 fAnalysisMacro(other.fAnalysisMacro),
168 fAnalysisSource(other.fAnalysisSource),
169 fAdditionalLibs(other.fAdditionalLibs),
170 fSplitMode(other.fSplitMode),
171 fAPIVersion(other.fAPIVersion),
172 fROOTVersion(other.fROOTVersion),
173 fAliROOTVersion(other.fAliROOTVersion),
648174cf 174 fExternalPackages(other.fExternalPackages),
c57f56b7 175 fUser(other.fUser),
176 fGridWorkingDir(other.fGridWorkingDir),
177 fGridDataDir(other.fGridDataDir),
178 fDataPattern(other.fDataPattern),
179 fGridOutputDir(other.fGridOutputDir),
180 fOutputArchive(other.fOutputArchive),
181 fOutputFiles(other.fOutputFiles),
182 fInputFormat(other.fInputFormat),
e7c71df0 183 fDatasetName(other.fDatasetName),
c57f56b7 184 fJDLName(other.fJDLName),
bb885a9e 185 fMergeExcludes(other.fMergeExcludes),
f965131e 186 fIncludePath(other.fIncludePath),
bb885a9e 187 fCloseSE(other.fCloseSE),
0df6ccf2 188 fFriendChainName(other.fFriendChainName),
c6cb3634 189 fJobTag(other.fJobTag),
648174cf 190 fOutputSingle(other.fOutputSingle),
5fce53f4 191 fRunPrefix(other.fRunPrefix),
4e5c5506 192 fInputFiles(0),
193 fPackages(0)
c57f56b7 194{
195// Copy ctor.
196 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 197 fRunRange[0] = other.fRunRange[0];
198 fRunRange[1] = other.fRunRange[1];
c57f56b7 199 if (other.fInputFiles) {
200 fInputFiles = new TObjArray();
201 TIter next(other.fInputFiles);
202 TObject *obj;
203 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
204 fInputFiles->SetOwner();
205 }
4e5c5506 206 if (other.fPackages) {
207 fPackages = new TObjArray();
208 TIter next(other.fPackages);
209 TObject *obj;
210 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
211 fPackages->SetOwner();
212 }
c57f56b7 213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::~AliAnalysisAlien()
217{
218// Destructor.
219 if (fGridJDL) delete fGridJDL;
220 if (fInputFiles) delete fInputFiles;
4e5c5506 221 if (fPackages) delete fPackages;
c57f56b7 222}
223
224//______________________________________________________________________________
225AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
226{
227// Assignment.
228 if (this != &other) {
229 AliAnalysisGrid::operator=(other);
230 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
231 fPrice = other.fPrice;
232 fTTL = other.fTTL;
233 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
234 fMaxInitFailed = other.fMaxInitFailed;
235 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 236 fNtestFiles = other.fNtestFiles;
a3e84053 237 fNrunsPerMaster = other.fNrunsPerMaster;
238 fMaxMergeFiles = other.fMaxMergeFiles;
239 fNsubmitted = other.fNsubmitted;
240 fProductionMode = other.fProductionMode;
cd11251e 241 fOutputToRunNo = other.fOutputToRunNo;
c57f56b7 242 fRunNumbers = other.fRunNumbers;
243 fExecutable = other.fExecutable;
0a1c1f7f 244 fExecutableCommand = other.fExecutableCommand;
c57f56b7 245 fArguments = other.fArguments;
631c0b05 246 fExecutableArgs = other.fExecutableArgs;
c57f56b7 247 fAnalysisMacro = other.fAnalysisMacro;
248 fAnalysisSource = other.fAnalysisSource;
249 fAdditionalLibs = other.fAdditionalLibs;
250 fSplitMode = other.fSplitMode;
251 fAPIVersion = other.fAPIVersion;
252 fROOTVersion = other.fROOTVersion;
253 fAliROOTVersion = other.fAliROOTVersion;
648174cf 254 fExternalPackages = other.fExternalPackages;
c57f56b7 255 fUser = other.fUser;
256 fGridWorkingDir = other.fGridWorkingDir;
257 fGridDataDir = other.fGridDataDir;
258 fDataPattern = other.fDataPattern;
259 fGridOutputDir = other.fGridOutputDir;
260 fOutputArchive = other.fOutputArchive;
261 fOutputFiles = other.fOutputFiles;
262 fInputFormat = other.fInputFormat;
e7c71df0 263 fDatasetName = other.fDatasetName;
c57f56b7 264 fJDLName = other.fJDLName;
bb885a9e 265 fMergeExcludes = other.fMergeExcludes;
f965131e 266 fIncludePath = other.fIncludePath;
bb885a9e 267 fCloseSE = other.fCloseSE;
0df6ccf2 268 fFriendChainName = other.fFriendChainName;
c6cb3634 269 fJobTag = other.fJobTag;
648174cf 270 fOutputSingle = other.fOutputSingle;
5fce53f4 271 fRunPrefix = other.fRunPrefix;
c57f56b7 272 if (other.fInputFiles) {
273 fInputFiles = new TObjArray();
274 TIter next(other.fInputFiles);
275 TObject *obj;
276 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
277 fInputFiles->SetOwner();
278 }
4e5c5506 279 if (other.fPackages) {
280 fPackages = new TObjArray();
281 TIter next(other.fPackages);
282 TObject *obj;
283 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
284 fPackages->SetOwner();
285 }
c57f56b7 286 }
287 return *this;
288}
289
f965131e 290//______________________________________________________________________________
291void AliAnalysisAlien::AddIncludePath(const char *path)
292{
293// Add include path in the remote analysis macro.
294 TString p(path);
295 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
296 else fIncludePath += Form("-I%s ", path);
297}
298
c57f56b7 299//______________________________________________________________________________
300void AliAnalysisAlien::AddRunNumber(Int_t run)
301{
302// Add a run number to the list of runs to be processed.
303 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 304 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 305}
306
ee75cfc3 307//______________________________________________________________________________
308void AliAnalysisAlien::AddRunNumber(const char* run)
309{
310// Add a run number to the list of runs to be processed.
311 if (fRunNumbers.Length()) fRunNumbers += " ";
312 fRunNumbers += run;
313}
314
c57f56b7 315//______________________________________________________________________________
316void AliAnalysisAlien::AddDataFile(const char *lfn)
317{
318// Adds a data file to the input to be analysed. The file should be a valid LFN
319// or point to an existing file in the alien workdir.
320 if (!fInputFiles) fInputFiles = new TObjArray();
321 fInputFiles->Add(new TObjString(lfn));
322}
648174cf 323
324//______________________________________________________________________________
325void AliAnalysisAlien::AddExternalPackage(const char *package)
326{
327// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
328 if (fExternalPackages) fExternalPackages += " ";
329 fExternalPackages += package;
330}
331
c57f56b7 332//______________________________________________________________________________
333Bool_t AliAnalysisAlien::Connect()
334{
335// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
336 if (gGrid && gGrid->IsConnected()) return kTRUE;
337 if (!gSystem->Getenv("alien_API_USER")) {
338 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
339 gSystem->Getenv("UID"));
340 return kFALSE;
341 }
342 if (!gGrid) {
343 Info("Connect", "Trying to connect to AliEn ...");
344 TGrid::Connect("alien://");
345 }
346 if (!gGrid || !gGrid->IsConnected()) {
347 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
348 return kFALSE;
349 }
350 fUser = gGrid->GetUser();
351 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
352 return kTRUE;
353}
354
355//______________________________________________________________________________
356void AliAnalysisAlien::CdWork()
357{
358// Check validity of alien workspace. Create directory if possible.
359 if (!Connect()) {
360 Error("CdWork", "Alien connection required");
361 return;
362 }
363 TString homedir = gGrid->GetHomeDirectory();
364 TString workdir = homedir + fGridWorkingDir;
923e2ca5 365 if (DirectoryExists(workdir)) {
366 gGrid->Cd(workdir);
367 return;
368 }
369 // Work directory not existing - create it
370 gGrid->Cd(homedir);
371 if (gGrid->Mkdir(workdir)) {
372 gGrid->Cd(fGridWorkingDir);
373 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
374 } else {
375 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
376 workdir.Data(), homedir.Data());
377 fGridWorkingDir = "";
378 }
c57f56b7 379}
380
381//______________________________________________________________________________
382Bool_t AliAnalysisAlien::CheckInputData()
383{
384// Check validity of input data. If necessary, create xml files.
a8739e8a 385 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 386 if (!fGridDataDir.Length()) {
387 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
388 return kFALSE;
389 }
390 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
391 return kTRUE;
c57f56b7 392 }
393 // Process declared files
394 Bool_t is_collection = kFALSE;
395 Bool_t is_xml = kFALSE;
396 Bool_t use_tags = kFALSE;
397 Bool_t checked = kFALSE;
398 CdWork();
399 TString file;
400 TString workdir = gGrid->GetHomeDirectory();
401 workdir += fGridWorkingDir;
402 if (fInputFiles) {
403 TObjString *objstr;
404 TIter next(fInputFiles);
405 while ((objstr=(TObjString*)next())) {
406 file = workdir;
407 file += "/";
408 file += objstr->GetString();
409 // Store full lfn path
410 if (FileExists(file)) objstr->SetString(file);
411 else {
412 file = objstr->GetName();
413 if (!FileExists(objstr->GetName())) {
414 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
415 objstr->GetName(), workdir.Data());
416 return kFALSE;
417 }
418 }
419 Bool_t iscoll, isxml, usetags;
420 CheckDataType(file, iscoll, isxml, usetags);
421 if (!checked) {
422 checked = kTRUE;
423 is_collection = iscoll;
424 is_xml = isxml;
425 use_tags = usetags;
426 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
427 } else {
428 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
429 Error("CheckInputData", "Some conflict was found in the types of inputs");
430 return kFALSE;
431 }
432 }
433 }
434 }
435 // Process requested run numbers
a8739e8a 436 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 437 // Check validity of alien data directory
438 if (!fGridDataDir.Length()) {
439 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
440 return kFALSE;
441 }
923e2ca5 442 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 443 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
444 return kFALSE;
445 }
446 if (is_collection) {
447 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
448 return kFALSE;
449 }
450
451 if (checked && !is_xml) {
452 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
453 return kFALSE;
454 }
455 // Check validity of run number(s)
456 TObjArray *arr;
457 TObjString *os;
319593fb 458 Int_t nruns = 0;
904f9f5f 459 TString schunk, schunk2;
c57f56b7 460 TString path;
461 if (!checked) {
462 checked = kTRUE;
463 use_tags = fDataPattern.Contains("tag");
464 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
465 }
466 if (use_tags != fDataPattern.Contains("tag")) {
467 Error("CheckInputData", "Cannot mix input files using/not using tags");
468 return kFALSE;
469 }
470 if (fRunNumbers.Length()) {
a8739e8a 471 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 472 arr = fRunNumbers.Tokenize(" ");
473 TIter next(arr);
474 while ((os=(TObjString*)next())) {
475 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 476 if (!DirectoryExists(path)) {
477 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 478 continue;
c57f56b7 479 }
480 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
481 TString msg = "\n##### file: ";
482 msg += path;
483 msg += " type: xml_collection;";
484 if (use_tags) msg += " using_tags: Yes";
485 else msg += " using_tags: No";
486 Info("CheckDataType", msg.Data());
319593fb 487 if (fNrunsPerMaster<2) {
d2a409b2 488 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 489 } else {
490 nruns++;
491 if (((nruns-1)%fNrunsPerMaster) == 0) {
492 schunk = os->GetString();
493 }
494 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
495 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 496 AddDataFile(schunk);
319593fb 497 }
c57f56b7 498 }
499 delete arr;
a8739e8a 500 } else {
501 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
502 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 503 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 504 if (!DirectoryExists(path)) {
505// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 506 continue;
507 }
5fce53f4 508 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 509 TString msg = "\n##### file: ";
510 msg += path;
511 msg += " type: xml_collection;";
512 if (use_tags) msg += " using_tags: Yes";
513 else msg += " using_tags: No";
514 Info("CheckDataType", msg.Data());
319593fb 515 if (fNrunsPerMaster<2) {
5fce53f4 516 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 517 } else {
518 nruns++;
519 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 520 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 521 }
904f9f5f 522 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 523 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 524 schunk += schunk2;
d2a409b2 525 AddDataFile(schunk);
319593fb 526 }
a8739e8a 527 }
904f9f5f 528 if (!fInputFiles) {
529 schunk += schunk2;
530 AddDataFile(schunk);
531 }
c57f56b7 532 }
533 return kTRUE;
534}
535
536//______________________________________________________________________________
537Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
538{
539// Create dataset for the grid data directory + run number.
540 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
541 if (!Connect()) {
542 Error("CreateDataset", "Cannot create dataset with no grid connection");
543 return kFALSE;
544 }
545
546 // Cd workspace
547 CdWork();
548 TString workdir = gGrid->GetHomeDirectory();
549 workdir += fGridWorkingDir;
550
551 // Compose the 'find' command arguments
552 TString command;
553 TString options = "-x collection ";
bb885a9e 554 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 555 TString conditions = "";
556
557 TString file;
558 TString path;
319593fb 559 Int_t nruns = 0;
904f9f5f 560 TString schunk, schunk2;
ab254fd1 561 TGridCollection *cbase=0, *cadd=0;
d2a409b2 562 if (!fRunNumbers.Length() && !fRunRange[0]) {
563 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
564 // Make a single data collection from data directory.
565 path = fGridDataDir;
923e2ca5 566 if (!DirectoryExists(path)) {
d2a409b2 567 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
568 return kFALSE;
569 }
923e2ca5 570// CdWork();
d2a409b2 571 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
572 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 573 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 574 command = "find ";
575 command += options;
576 command += path;
577 command += " ";
578 command += pattern;
579 command += conditions;
84fcd93f 580 printf("command: %s\n", command.Data());
d2a409b2 581 TGridResult *res = gGrid->Command(command);
582 if (res) delete res;
583 // Write standard output to file
584 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
585 }
586 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
587 // Copy xml file to alien space
588 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
589 if (!FileExists(file)) {
590 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
591 return kFALSE;
592 }
593 // Update list of files to be processed.
594 }
595 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
596 return kTRUE;
597 }
c57f56b7 598 // Several runs
a8739e8a 599 if (fRunNumbers.Length()) {
600 TObjArray *arr = fRunNumbers.Tokenize(" ");
601 TObjString *os;
602 TIter next(arr);
603 while ((os=(TObjString*)next())) {
604 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 605 if (!DirectoryExists(path)) continue;
606// CdWork();
a8739e8a 607 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
608 else file = Form("%s.xml", os->GetString().Data());
319593fb 609 // If local collection file does not exist, create it via 'find' command.
610 if (gSystem->AccessPathName(file)) {
611 command = "find ";
612 command += options;
613 command += path;
614 command += pattern;
615 command += conditions;
616 TGridResult *res = gGrid->Command(command);
617 if (res) delete res;
618 // Write standard output to file
619 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
620 }
a8739e8a 621 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 622 // Check if there is one run per master job.
623 if (fNrunsPerMaster<2) {
624 if (FileExists(file)) {
625 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
626 continue;
627 }
628 // Copy xml file to alien space
629 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
630 if (!FileExists(file)) {
631 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
632 delete arr;
633 return kFALSE;
634 }
635 } else {
636 nruns++;
637 if (((nruns-1)%fNrunsPerMaster) == 0) {
638 schunk = os->GetString();
639 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
640 } else {
641 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 642 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 643 cbase->Add(cadd);
644 delete cadd;
645 }
646 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
647 continue;
648 }
649 schunk += Form("_%s.xml", os->GetString().Data());
650 if (FileExists(schunk)) {
651 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
652 continue;
653 }
84fcd93f 654 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 655 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 656 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 657 if (!FileExists(schunk)) {
658 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
659 delete arr;
660 return kFALSE;
661 }
662 }
a8739e8a 663 }
664 delete arr;
665 } else {
666 // Process a full run range.
667 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 668 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 669 if (!DirectoryExists(path)) continue;
670// CdWork();
a8739e8a 671 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 672 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
319593fb 673 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 674 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
675// gGrid->Rm(file);
676 continue;
677 }
319593fb 678 // If local collection file does not exist, create it via 'find' command.
679 if (gSystem->AccessPathName(file)) {
680 command = "find ";
681 command += options;
682 command += path;
683 command += pattern;
684 command += conditions;
685 TGridResult *res = gGrid->Command(command);
686 if (res) delete res;
687 // Write standard output to file
688 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
689 }
a8739e8a 690 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 691 // Check if there is one run per master job.
692 if (fNrunsPerMaster<2) {
693 if (FileExists(file)) {
694 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
695 continue;
696 }
697 // Copy xml file to alien space
698 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
699 if (!FileExists(file)) {
700 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
701 return kFALSE;
702 }
703 } else {
704 nruns++;
95e5b448 705 // Check if the collection for the chunk exist locally.
706 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
707 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 708 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 709 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 710 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 711 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
712 } else {
713 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
714 cbase->Add(cadd);
715 delete cadd;
716 }
904f9f5f 717 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
718 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 719 continue;
720 }
904f9f5f 721 schunk = schunk2;
319593fb 722 if (FileExists(schunk)) {
723 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
724 continue;
725 }
84fcd93f 726 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 727 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 728 if (FileExists(schunk)) {
729 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
730 continue;
731 }
319593fb 732 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
733 if (!FileExists(schunk)) {
734 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
735 return kFALSE;
736 }
737 }
c57f56b7 738 }
a8739e8a 739 }
c57f56b7 740 return kTRUE;
741}
742
743//______________________________________________________________________________
744Bool_t AliAnalysisAlien::CreateJDL()
745{
746// Generate a JDL file according to current settings. The name of the file is
747// specified by fJDLName.
748 Bool_t error = kFALSE;
749 TObjArray *arr = 0;
750 Bool_t copy = kTRUE;
751 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
752 Bool_t generate = kTRUE;
753 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
754 if (!Connect()) {
755 Error("CreateJDL", "Alien connection required");
756 return kFALSE;
757 }
758 // Check validity of alien workspace
759 CdWork();
760 TString workdir = gGrid->GetHomeDirectory();
761 workdir += fGridWorkingDir;
762 if (generate) {
763 TObjString *os;
764 if (!fInputFiles) {
765 Error("CreateJDL()", "Define some input files for your analysis.");
766 error = kTRUE;
767 }
768 // Compose list of input files
769 // Check if output files were defined
770 if (!fOutputFiles.Length()) {
771 Error("CreateJDL", "You must define at least one output file");
772 error = kTRUE;
773 }
774 // Check if an output directory was defined and valid
775 if (!fGridOutputDir.Length()) {
776 Error("CreateJDL", "You must define AliEn output directory");
777 error = kTRUE;
778 } else {
d2a409b2 779 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 780 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 781 if (gGrid->Mkdir(fGridOutputDir)) {
782 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
783 } else {
784 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 785 // error = kTRUE;
c57f56b7 786 }
787 }
788 gGrid->Cd(workdir);
789 }
790 // Exit if any error up to now
791 if (error) return kFALSE;
792 // Set JDL fields
793 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 794 fGridJDL->SetExecutable(fExecutable);
b5b9dee8 795 if (!fArguments.IsNull())
796 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
c57f56b7 797// fGridJDL->SetTTL((UInt_t)fTTL);
798 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
799 if (fMaxInitFailed > 0)
800 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
801 if (fSplitMaxInputFileNumber > 0)
802 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
803 if (fSplitMode.Length())
804 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
805// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
806 if (fAliROOTVersion.Length())
807 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
808 if (fROOTVersion.Length())
809 fGridJDL->AddToPackages("ROOT", fROOTVersion);
810 if (fAPIVersion.Length())
811 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
648174cf 812 if (!fExternalPackages.IsNull()) {
813 arr = fExternalPackages.Tokenize(" ");
814 TIter next(arr);
815 while ((os=(TObjString*)next())) {
816 TString pkgname = os->GetString();
817 Int_t index = pkgname.Index("::");
818 TString pkgversion = pkgname(index+2, pkgname.Length());
819 pkgname.Remove(index);
820 fGridJDL->AddToPackages(pkgname, pkgversion);
821 }
822 delete arr;
823 }
c57f56b7 824 fGridJDL->SetInputDataListFormat(fInputFormat);
825 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 826 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 827 TString analysisFile = fExecutable;
828 analysisFile.ReplaceAll(".sh", ".root");
829 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 830 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
831 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
832 if (fAdditionalLibs.Length()) {
833 arr = fAdditionalLibs.Tokenize(" ");
834 TIter next(arr);
835 while ((os=(TObjString*)next())) {
836 if (os->GetString().Contains(".so")) continue;
837 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
838 }
839 delete arr;
840 }
4e5c5506 841 if (fPackages) {
842 TIter next(fPackages);
843 TObject *obj;
844 while ((obj=next()))
845 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
846 }
c57f56b7 847 if (fOutputArchive.Length()) {
848 arr = fOutputArchive.Tokenize(" ");
849 TIter next(arr);
850 while ((os=(TObjString*)next()))
43da816a 851 if (!os->GetString().Contains("@") && fCloseSE.Length())
852 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
853 else
854 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 855 delete arr;
856 }
c57f56b7 857 arr = fOutputFiles.Tokenize(" ");
858 TIter next(arr);
43da816a 859 while ((os=(TObjString*)next())) {
e1eaf596 860 // Ignore ouputs in jdl that are also in outputarchive
861 TString sout = os->GetString();
862 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
863 if (fOutputArchive.Contains(sout)) continue;
43da816a 864 if (!os->GetString().Contains("@") && fCloseSE.Length())
865 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
866 else
867 fGridJDL->AddToOutputSandbox(os->GetString());
868 }
c57f56b7 869 delete arr;
870// fGridJDL->SetPrice((UInt_t)fPrice);
871 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 872 TString validationScript = fExecutable;
873 validationScript.ReplaceAll(".sh", "_validation.sh");
874 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 875 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 876 // Write a jdl with 2 input parameters: collection name and output dir name.
877 WriteJDL(copy);
c57f56b7 878 }
879 // Copy jdl to grid workspace
a8739e8a 880 if (copy) {
b5e4aaa7 881 // Check if an output directory was defined and valid
882 if (!fGridOutputDir.Length()) {
883 Error("CreateJDL", "You must define AliEn output directory");
884 return kFALSE;
885 } else {
886 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
887 if (!DirectoryExists(fGridOutputDir)) {
888 if (gGrid->Mkdir(fGridOutputDir)) {
889 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
890 } else {
891 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
892 return kFALSE;
893 }
894 }
895 gGrid->Cd(workdir);
896 }
648174cf 897 if (TestBit(AliAnalysisGrid::kSubmit)) {
898 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
899 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
900 if (fProductionMode)
901 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
902 if (FileExists(locjdl)) gGrid->Rm(locjdl);
903 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
904 }
c57f56b7 905 if (fAdditionalLibs.Length()) {
e7c71df0 906 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 907 TObjString *os;
908 TIter next(arr);
909 while ((os=(TObjString*)next())) {
c57f56b7 910 if (os->GetString().Contains(".so")) continue;
4e5c5506 911 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 912 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
913 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
914 }
915 delete arr;
916 }
4e5c5506 917 if (fPackages) {
918 TIter next(fPackages);
919 TObject *obj;
920 while ((obj=next())) {
921 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
922 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
923 }
924 }
c57f56b7 925 }
926 return kTRUE;
927}
928
a8739e8a 929//______________________________________________________________________________
d2a409b2 930Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 931{
932// Writes one or more JDL's corresponding to findex. If findex is negative,
933// all run numbers are considered in one go (jdl). For non-negative indices
934// they correspond to the indices in the array fInputFiles.
935 if (!fInputFiles) return kFALSE;
936 TObjString *os;
a8739e8a 937 TString workdir = gGrid->GetHomeDirectory();
938 workdir += fGridWorkingDir;
d2a409b2 939
940 if (!fRunNumbers.Length() && !fRunRange[0]) {
941 // One jdl with no parameters in case input data is specified by name.
a8739e8a 942 TIter next(fInputFiles);
943 while ((os=(TObjString*)next()))
944 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
648174cf 945 if (!fOutputSingle.IsNull())
946 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
947 else
948 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 949 } else {
d2a409b2 950 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
951 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
cd11251e 952 if (!fOutputSingle.IsNull()) {
953 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
954 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()));
955 } else {
956 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
957 }
a8739e8a 958 }
959
960
961 // Generate the JDL as a string
962 TString sjdl = fGridJDL->Generate();
963 Int_t index;
964 index = sjdl.Index("Executable");
965 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
966 index = sjdl.Index("Split ");
648174cf 967 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
a8739e8a 968 index = sjdl.Index("SplitMaxInputFileNumber");
969 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
970 index = sjdl.Index("InputDataCollection");
971 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 972 index = sjdl.Index("InputFile");
973 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
974 index = sjdl.Index("InputDataList ");
975 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
976 index = sjdl.Index("InputDataListFormat");
977 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
978 index = sjdl.Index("Price");
979 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
980 index = sjdl.Index("Requirements");
981 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
982 index = sjdl.Index("Packages");
983 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
984 index = sjdl.Index("User =");
985 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
986 index = sjdl.Index("TTL");
987 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
988 index = sjdl.Index("OutputFile");
989 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
990 index = sjdl.Index("OutputDir");
991 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
992 index = sjdl.Index("OutputArchive");
993 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
994 index = sjdl.Index("MaxInitFailed");
995 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
996 index = sjdl.Index("MasterResubmitThreshold");
997 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
998 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
999 index = sjdl.Index("Validationcommand");
1000 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
1001 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1002 sjdl.ReplaceAll("(member", "\n (member");
1003 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1004 sjdl.ReplaceAll("{", "{\n ");
1005 sjdl.ReplaceAll("};", "\n};");
1006 sjdl.ReplaceAll("{\n \n", "{\n");
1007 sjdl.ReplaceAll("\n\n", "\n");
1008 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
1009 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1010 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1011 index = sjdl.Index("JDLVariables");
1012 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
1013 // Write jdl to file
a8739e8a 1014 ofstream out;
d2a409b2 1015 out.open(fJDLName.Data(), ios::out);
a8739e8a 1016 if (out.bad()) {
d2a409b2 1017 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1018 return kFALSE;
1019 }
1020 out << sjdl << endl;
1021
1022 // Copy jdl to grid workspace
1023 if (!copy) {
1024 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1025 } else {
d2a409b2 1026 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1027 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 1028 if (fProductionMode)
b5fe9cba 1029 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 1030 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 1031 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 1032 }
1033 return kTRUE;
1034}
1035
c57f56b7 1036//______________________________________________________________________________
5513444a 1037Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1038{
1039// Returns true if file exists.
5513444a 1040 if (!gGrid) return kFALSE;
c57f56b7 1041 TGridResult *res = gGrid->Ls(lfn);
1042 if (!res) return kFALSE;
1043 TMap *map = dynamic_cast<TMap*>(res->At(0));
1044 if (!map) {
1045 delete res;
1046 return kFALSE;
1047 }
1048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1049 if (!objs || !objs->GetString().Length()) {
1050 delete res;
1051 return kFALSE;
1052 }
1053 delete res;
1054 return kTRUE;
1055}
1056
923e2ca5 1057//______________________________________________________________________________
1058Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1059{
1060// Returns true if directory exists. Can be also a path.
1061 if (!gGrid) return kFALSE;
1062 // Check if dirname is a path
1063 TString dirstripped = dirname;
1064 dirstripped = dirstripped.Strip();
1065 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1066 TString dir = gSystem->BaseName(dirstripped);
1067 dir += "/";
1068 TString path = gSystem->DirName(dirstripped);
1069 TGridResult *res = gGrid->Ls(path, "-F");
1070 if (!res) return kFALSE;
1071 TIter next(res);
1072 TMap *map;
1073 TObject *obj;
1074 while ((map=dynamic_cast<TMap*>(next()))) {
1075 obj = map->GetValue("name");
1076 if (!obj) break;
1077 if (dir == obj->GetName()) {
1078 delete res;
1079 return kTRUE;
1080 }
1081 }
1082 delete res;
1083 return kFALSE;
1084}
1085
c57f56b7 1086//______________________________________________________________________________
1087void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1088{
1089// Check input data type.
1090 is_collection = kFALSE;
1091 is_xml = kFALSE;
1092 use_tags = kFALSE;
1093 if (!gGrid) {
1094 Error("CheckDataType", "No connection to grid");
1095 return;
1096 }
1097 is_collection = IsCollection(lfn);
1098 TString msg = "\n##### file: ";
1099 msg += lfn;
1100 if (is_collection) {
1101 msg += " type: raw_collection;";
1102 // special treatment for collections
1103 is_xml = kFALSE;
1104 // check for tag files in the collection
1105 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1106 if (!res) {
1107 msg += " using_tags: No (unknown)";
1108 Info("CheckDataType", msg.Data());
1109 return;
1110 }
1111 const char* typeStr = res->GetKey(0, "origLFN");
1112 if (!typeStr || !strlen(typeStr)) {
1113 msg += " using_tags: No (unknown)";
1114 Info("CheckDataType", msg.Data());
1115 return;
1116 }
1117 TString file = typeStr;
1118 use_tags = file.Contains(".tag");
1119 if (use_tags) msg += " using_tags: Yes";
1120 else msg += " using_tags: No";
1121 Info("CheckDataType", msg.Data());
1122 return;
1123 }
1124 TString slfn(lfn);
1125 slfn.ToLower();
1126 is_xml = slfn.Contains(".xml");
1127 if (is_xml) {
1128 // Open xml collection and check if there are tag files inside
1129 msg += " type: xml_collection;";
1130 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1131 if (!coll) {
1132 msg += " using_tags: No (unknown)";
1133 Info("CheckDataType", msg.Data());
1134 return;
1135 }
1136 TMap *map = coll->Next();
1137 if (!map) {
1138 msg += " using_tags: No (unknown)";
1139 Info("CheckDataType", msg.Data());
1140 return;
1141 }
1142 map = (TMap*)map->GetValue("");
1143 TString file;
1144 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1145 use_tags = file.Contains(".tag");
1146 delete coll;
1147 if (use_tags) msg += " using_tags: Yes";
1148 else msg += " using_tags: No";
1149 Info("CheckDataType", msg.Data());
1150 return;
1151 }
1152 use_tags = slfn.Contains(".tag");
1153 if (slfn.Contains(".root")) msg += " type: root file;";
1154 else msg += " type: unhnown file;";
1155 if (use_tags) msg += " using_tags: Yes";
1156 else msg += " using_tags: No";
1157 Info("CheckDataType", msg.Data());
1158}
1159
4e5c5506 1160//______________________________________________________________________________
1161void AliAnalysisAlien::EnablePackage(const char *package)
1162{
1163// Enables a par file supposed to exist in the current directory.
1164 TString pkg(package);
1165 pkg.ReplaceAll(".par", "");
1166 pkg += ".par";
1167 if (gSystem->AccessPathName(pkg)) {
1168 Error("EnablePackage", "Package %s not found", pkg.Data());
1169 return;
1170 }
1171 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1172 Info("EnablePackage", "AliEn plugin will use .par packages");
1173 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1174 if (!fPackages) {
1175 fPackages = new TObjArray();
1176 fPackages->SetOwner();
1177 }
1178 fPackages->Add(new TObjString(pkg));
1179}
1180
d2a409b2 1181//______________________________________________________________________________
1182const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1183{
1184// Get job status for all jobs with jobid>jobidstart.
1185 static char mstatus[20];
1186 mstatus[0] = '\0';
1187 nrunning = 0;
1188 nwaiting = 0;
1189 nerror = 0;
1190 ndone = 0;
1191 TGridJobStatusList *list = gGrid->Ps("");
1192 if (!list) return mstatus;
1193 Int_t nentries = list->GetSize();
1194 TGridJobStatus *status;
1195 Int_t pid;
1196 for (Int_t ijob=0; ijob<nentries; ijob++) {
1197 status = (TGridJobStatus *)list->At(ijob);
1198 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1199 if (pid<jobidstart) continue;
1200 if (pid == lastid) {
1201 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1202 }
1203 switch (status->GetStatus()) {
1204 case TGridJobStatus::kWAITING:
1205 nwaiting++; break;
1206 case TGridJobStatus::kRUNNING:
1207 nrunning++; break;
1208 case TGridJobStatus::kABORTED:
1209 case TGridJobStatus::kFAIL:
1210 case TGridJobStatus::kUNKNOWN:
1211 nerror++; break;
1212 case TGridJobStatus::kDONE:
1213 ndone++;
1214 }
1215 }
1216 list->Delete();
1217 delete list;
1218 return mstatus;
1219}
1220
c57f56b7 1221//______________________________________________________________________________
1222Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1223{
1224// Returns true if file is a collection. Functionality duplicated from
1225// TAlien::Type() because we don't want to directly depend on TAlien.
1226 if (!gGrid) {
1227 Error("IsCollection", "No connection to grid");
1228 return kFALSE;
1229 }
1230 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1231 if (!res) return kFALSE;
1232 const char* typeStr = res->GetKey(0, "type");
1233 if (!typeStr || !strlen(typeStr)) return kFALSE;
1234 if (!strcmp(typeStr, "collection")) return kTRUE;
1235 delete res;
1236 return kFALSE;
1237}
1238
fe2d7fc2 1239//______________________________________________________________________________
1240Bool_t AliAnalysisAlien::IsSingleOutput() const
1241{
1242// Check if single-ouput option is on.
1243 return (!fOutputSingle.IsNull());
1244}
1245
16a4353c 1246//______________________________________________________________________________
1247void AliAnalysisAlien::Print(Option_t *) const
1248{
1249// Print current plugin settings.
84fcd93f 1250 printf("### AliEn analysis plugin current settings ###\n");
1251 printf("= Production mode:______________________________ %d\n", fProductionMode);
1252 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1253 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1254 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1255 if (fUser.Length())
84fcd93f 1256 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1257 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1258 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1259 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1260 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1261 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1262 if (fRunNumbers.Length())
84fcd93f 1263 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1264 if (fRunRange[0])
5fce53f4 1265 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1266 if (!fRunRange[0] && !fRunNumbers.Length()) {
1267 TIter next(fInputFiles);
1268 TObject *obj;
1269 TString list;
1270 while ((obj=next())) list += obj->GetName();
84fcd93f 1271 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1272 }
1273 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1274 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1275 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1276 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1277 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1278 printf("=====================================================================\n");
1279 printf("= Job price: ___________________________________ %d\n", fPrice);
1280 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1281 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1282 if (fMaxInitFailed>0)
84fcd93f 1283 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1284 if (fMasterResubmitThreshold>0)
84fcd93f 1285 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1286 if (fNrunsPerMaster>0)
84fcd93f 1287 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1288 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1289 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1290 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1291 if (fArguments.Length())
84fcd93f 1292 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1293 if (fExecutableArgs.Length())
1294 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1295 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1296 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1297 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1298 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1299 if (fDatasetName)
84fcd93f 1300 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1301 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1302 if (fIncludePath.Data())
84fcd93f 1303 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1304 if (fCloseSE.Length())
84fcd93f 1305 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1306 if (fFriendChainName.Length())
84fcd93f 1307 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1308 if (fPackages) {
1309 TIter next(fPackages);
1310 TObject *obj;
1311 TString list;
1312 while ((obj=next())) list += obj->GetName();
84fcd93f 1313 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1314 }
1315}
1316
c57f56b7 1317//______________________________________________________________________________
1318void AliAnalysisAlien::SetDefaults()
1319{
1320// Set default values for everything. What cannot be filled will be left empty.
1321 if (fGridJDL) delete fGridJDL;
1322 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1323 fPrice = 1;
1324 fTTL = 30000;
1325 fSplitMaxInputFileNumber = 100;
1326 fMaxInitFailed = 0;
1327 fMasterResubmitThreshold = 0;
bb885a9e 1328 fNtestFiles = 10;
a8739e8a 1329 fRunRange[0] = 0;
1330 fRunRange[1] = 0;
319593fb 1331 fNrunsPerMaster = 1;
16a4353c 1332 fMaxMergeFiles = 100;
c57f56b7 1333 fRunNumbers = "";
1334 fExecutable = "analysis.sh";
0a1c1f7f 1335 fExecutableCommand = "root -b -q";
c57f56b7 1336 fArguments = "";
631c0b05 1337 fExecutableArgs = "";
c57f56b7 1338 fAnalysisMacro = "myAnalysis.C";
1339 fAnalysisSource = "";
1340 fAdditionalLibs = "";
1341 fSplitMode = "se";
1342 fAPIVersion = "";
1343 fROOTVersion = "";
1344 fAliROOTVersion = "";
1345 fUser = ""; // Your alien user name
1346 fGridWorkingDir = "";
1347 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1348 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1349 fFriendChainName = "";
c57f56b7 1350 fGridOutputDir = "output";
1351 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1352 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1353 fInputFormat = "xml-single";
1354 fJDLName = "analysis.jdl";
c6cb3634 1355 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1356 fMergeExcludes = "";
c57f56b7 1357}
1358
1359//______________________________________________________________________________
1360Bool_t AliAnalysisAlien::MergeOutputs()
1361{
1362// Merge analysis outputs existing in the AliEn space.
1363 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1364 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1365 if (!Connect()) {
1366 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1367 return kFALSE;
1368 }
1369 // Get the output path
d2a409b2 1370 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1371 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1372 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1373 return kFALSE;
1374 }
1375 if (!fOutputFiles.Length()) {
1376 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1377 return kFALSE;
1378 }
1379 TObjArray *list = fOutputFiles.Tokenize(" ");
1380 TIter next(list);
1381 TObjString *str;
1382 TString command;
1383 TString output_file;
16a4353c 1384 TString output_chunk;
1385 TString previous_chunk;
1386 Int_t count_chunk = 0;
1387 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1388 Bool_t merged = kTRUE;
1389 while((str=(TObjString*)next())) {
1390 output_file = str->GetString();
1391 Int_t index = output_file.Index("@");
1392 if (index > 0) output_file.Remove(index);
319593fb 1393 // Skip already merged outputs
1394 if (!gSystem->AccessPathName(output_file)) {
1395 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1396 continue;
1397 }
bb885a9e 1398 if (fMergeExcludes.Length() &&
1399 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1400 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1401 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1402 printf("command: %s\n", command.Data());
c57f56b7 1403 TGridResult *res = gGrid->Command(command);
1404 if (!res) continue;
1405 TFileMerger *fm = 0;
1406 TIter nextmap(res);
ab254fd1 1407 TMap *map = 0;
16a4353c 1408 previous_chunk = "";
1409 count_chunk = 0;
319593fb 1410 // Check if there is a merge operation to resume
1411 output_chunk = output_file;
1412 output_chunk.ReplaceAll(".root", "_*.root");
1413 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1414 while (1) {
1415 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1416 if (!map) {
1417 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1418 delete res;
1419 return kFALSE;
1420 }
1421 output_chunk = output_file;
1422 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1423 printf("%s\n", output_chunk.Data());
319593fb 1424 count_chunk++;
1425 if (gSystem->AccessPathName(output_chunk)) continue;
1426 // Merged file with chunks up to <count_chunk> found
84fcd93f 1427 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1428 previous_chunk = output_chunk;
1429 break;
1430 }
1431 }
16a4353c 1432 count_zero = fMaxMergeFiles;
c57f56b7 1433 while ((map=(TMap*)nextmap())) {
16a4353c 1434 // Loop 'find' results and get next LFN
1435 if (count_zero == fMaxMergeFiles) {
1436 // First file in chunk - create file merger and add previous chunk if any.
1437 fm = new TFileMerger(kFALSE);
1438 fm->SetFastMethod(kTRUE);
1439 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1440 output_chunk = output_file;
1441 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1442 }
1443 // If last file found, put merged results in the output file
1444 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1445 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1446 if (!objs || !objs->GetString().Length()) {
16a4353c 1447 // Nothing found - skip this output
c57f56b7 1448 delete res;
16a4353c 1449 delete fm;
1450 break;
1451 }
1452 // Add file to be merged and decrement chunk counter.
1453 fm->AddFile(objs->GetString());
1454 count_zero--;
1455 if (count_zero==0 || map == res->Last()) {
1456 fm->OutputFile(output_chunk);
1457 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1458 // Nothing found - skip this output
1459 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1460 delete res;
1461 delete fm;
1462 break;
1463 }
1464 // Merge the outputs, then go to next chunk
1465 if (!fm->Merge()) {
1466 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1467 delete res;
1468 delete fm;
1469 merged = kFALSE;
1470 break;
1471 } else {
1472 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1473 gSystem->Unlink(previous_chunk);
16a4353c 1474 }
1475 if (map == res->Last()) {
1476 delete res;
1477 delete fm;
1478 break;
1479 }
1480 count_chunk++;
1481 count_zero = fMaxMergeFiles;
1482 previous_chunk = output_chunk;
c57f56b7 1483 }
c57f56b7 1484 }
c57f56b7 1485 }
1486 if (!merged) {
1487 Error("MergeOutputs", "Terminate() will NOT be executed");
1488 }
1489 return merged;
1490}
1491
bb885a9e 1492//______________________________________________________________________________
1493void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1494{
1495// Use the output files connected to output containers from the analysis manager
1496// rather than the files defined by SetOutputFiles
1497 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1498 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1499 analysis manager");
1500 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1501}
1502
5513444a 1503//______________________________________________________________________________
1504Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1505{
1506// Start remote grid analysis.
1507
43da816a 1508 // Check if output files have to be taken from the analysis manager
1509 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1510 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1511 if (!mgr || !mgr->IsInitialized()) {
1512 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1513 return kFALSE;
43da816a 1514 }
1515 fOutputFiles = "";
1516 TIter next(mgr->GetOutputs());
1517 AliAnalysisDataContainer *output;
1518 while ((output=(AliAnalysisDataContainer*)next())) {
1519 const char *filename = output->GetFileName();
1520 if (!(strcmp(filename, "default"))) {
1521 if (!mgr->GetOutputEventHandler()) continue;
1522 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1523 }
84fcd93f 1524 if (fOutputFiles.Contains(filename)) continue;
43da816a 1525 if (fOutputFiles.Length()) fOutputFiles += " ";
1526 fOutputFiles += filename;
1527 }
c07b9ce2 1528 // Add extra files registered to the analysis manager
1529 if (mgr->GetExtraFiles().Length()) {
1530 if (fOutputFiles.Length()) fOutputFiles += " ";
1531 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1532 }
43da816a 1533 }
f7b1cbc2 1534// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1535 if (TestBit(AliAnalysisGrid::kOffline)) {
1536 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1537 \n there nor any job run. You can revise the JDL and analysis \
1538 \n macro then run the same in \"submit\" mode.");
1539 } else if (TestBit(AliAnalysisGrid::kTest)) {
1540 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1541 \n dataset.");
1542 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1543 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1544 \n space and job submitted.");
1545 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1546 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1547 return kTRUE;
c57f56b7 1548 } else {
1549 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1550 }
1551
1552 if (!Connect()) {
1553 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1554 return kFALSE;
16a4353c 1555 }
1556 Print();
c57f56b7 1557 if (!CheckInputData()) {
1558 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1559 return kFALSE;
c57f56b7 1560 }
1561 CreateDataset(fDataPattern);
1562 WriteAnalysisFile();
1563 WriteAnalysisMacro();
1564 WriteExecutable();
1565 WriteValidationScript();
5513444a 1566 if (!CreateJDL()) return kFALSE;
1567 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1568 if (TestBit(AliAnalysisGrid::kTest)) {
1569 // Locally testing the analysis
1570 Info("StartAnalysis", "\n_______________________________________________________________________ \
1571 \n Running analysis script in a daughter shell as on a worker node \
1572 \n_______________________________________________________________________");
1573 TObjArray *list = fOutputFiles.Tokenize(" ");
1574 TIter next(list);
1575 TObjString *str;
1576 TString output_file;
1577 while((str=(TObjString*)next())) {
1578 output_file = str->GetString();
1579 Int_t index = output_file.Index("@");
1580 if (index > 0) output_file.Remove(index);
43da816a 1581 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1582 }
1583 delete list;
1584 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1585 TString validationScript = fExecutable;
1586 validationScript.ReplaceAll(".sh", "_validation.sh");
1587 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1588// gSystem->Exec("cat stdout");
5513444a 1589 return kFALSE;
c57f56b7 1590 }
5513444a 1591 // Check if submitting is managed by LPM manager
a3e84053 1592 if (fProductionMode) {
5513444a 1593 TString prodfile = fJDLName;
1594 prodfile.ReplaceAll(".jdl", ".prod");
1595 WriteProductionFile(prodfile);
1596 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1597 return kFALSE;
1598 }
a8739e8a 1599 // Submit AliEn job(s)
d2a409b2 1600 gGrid->Cd(fGridOutputDir);
a8739e8a 1601 TGridResult *res;
c57f56b7 1602 TString jobID = "";
d2a409b2 1603 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1604 // Submit a given xml or a set of runs
a8739e8a 1605 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1606 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1607 if (res) {
1608 const char *cjobId = res->GetKey(0,"jobId");
1609 if (!cjobId) {
1610 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1611 return kFALSE;
a8739e8a 1612 } else {
1613 Info("StartAnalysis", "\n_______________________________________________________________________ \
1614 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1615 \n_______________________________________________________________________",
1616 fJDLName.Data(), cjobId);
1617 jobID = cjobId;
1618 }
1619 delete res;
1620 }
1621 } else {
d2a409b2 1622 // Submit for a range of enumeration of runs.
1623 Submit();
c57f56b7 1624 }
a8739e8a 1625
c57f56b7 1626 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1627 \n You may exit at any time and terminate the job later using the option <terminate> \
1628 \n ##################################################################################", jobID.Data());
bb885a9e 1629 gSystem->Exec("aliensh");
5513444a 1630 return kTRUE;
c57f56b7 1631}
1632
d2a409b2 1633//______________________________________________________________________________
1634void AliAnalysisAlien::Submit()
1635{
1636// Submit all master jobs.
1637 Int_t nmasterjobs = fInputFiles->GetEntries();
1638 Long_t tshoot = gSystem->Now();
1639 if (!fNsubmitted) SubmitNext();
1640 while (fNsubmitted < nmasterjobs) {
1641 Long_t now = gSystem->Now();
1642 if ((now-tshoot)>30000) {
1643 tshoot = now;
1644 SubmitNext();
1645 }
1646 }
1647}
1648
1649//______________________________________________________________________________
1650void AliAnalysisAlien::SubmitNext()
1651{
1652// Submit next bunch of master jobs if the queue is free.
1653 static Bool_t iscalled = kFALSE;
1654 static Int_t firstmaster = 0;
1655 static Int_t lastmaster = 0;
1656 static Int_t npermaster = 0;
1657 if (iscalled) return;
1658 iscalled = kTRUE;
1659 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1660 Int_t ntosubmit = 0;
1661 TGridResult *res;
1662 TString jobID = "";
1663 if (!fNsubmitted) ntosubmit = 1;
1664 else {
1665 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1666 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1667 // If last master not split, just return
1668 if (status != "SPLIT") {iscalled = kFALSE; return;}
1669 // No more than 100 waiting jobs
1670 if (nwaiting>100) {iscalled = kFALSE; return;}
1671 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1672 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1673 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1674 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1675 }
1676 Int_t nmasterjobs = fInputFiles->GetEntries();
1677 for (Int_t i=0; i<ntosubmit; i++) {
1678 // Submit for a range of enumeration of runs.
1679 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1680 TString query;
cd11251e 1681 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
1682 runOutDir.ReplaceAll(".xml", "");
1683 if (fOutputToRunNo)
1684 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
1685 else
1686 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1687 printf("********* %s\n",query.Data());
d2a409b2 1688 res = gGrid->Command(query);
1689 if (res) {
98ca124f 1690 TString cjobId1 = res->GetKey(0,"jobId");
1691 if (!cjobId1.Length()) {
d2a409b2 1692 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1693 iscalled = kFALSE;
1694 return;
1695 } else {
1696 Info("StartAnalysis", "\n_______________________________________________________________________ \
1697 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1698 \n_______________________________________________________________________",
98ca124f 1699 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1700 jobID += cjobId1;
1701 jobID += " ";
98ca124f 1702 lastmaster = cjobId1.Atoi();
d2a409b2 1703 if (!firstmaster) firstmaster = lastmaster;
1704 fNsubmitted++;
1705 }
1706 delete res;
1707 }
1708 }
1709 iscalled = kFALSE;
1710}
1711
c57f56b7 1712//______________________________________________________________________________
1713void AliAnalysisAlien::WriteAnalysisFile()
1714{
f10e8481 1715// Write current analysis manager into the file <analysisFile>
1716 TString analysisFile = fExecutable;
1717 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1718 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1719 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1720 if (!mgr || !mgr->IsInitialized()) {
1721 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1722 return;
1723 }
1724 // Check analysis type
1725 TObject *handler;
1726 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1727 handler = (TObject*)mgr->GetInputEventHandler();
1728 if (handler) {
1729 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1730 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1731 }
1732 TDirectory *cdir = gDirectory;
f10e8481 1733 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1734 if (file) {
fe2d7fc2 1735 // Skip task Terminate calls for the grid job
1736 mgr->SetSkipTerminate(kTRUE);
1737 // Unless merging makes no sense
1738 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 1739 mgr->Write();
1740 delete file;
fe2d7fc2 1741 // Enable termination for local jobs
1742 mgr->SetSkipTerminate(kFALSE);
c57f56b7 1743 }
1744 if (cdir) cdir->cd();
f10e8481 1745 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1746 }
1747 Bool_t copy = kTRUE;
1748 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1749 if (copy) {
1750 CdWork();
1751 TString workdir = gGrid->GetHomeDirectory();
1752 workdir += fGridWorkingDir;
f10e8481 1753 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1754 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1755 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1756 }
1757}
1758
1759//______________________________________________________________________________
1760void AliAnalysisAlien::WriteAnalysisMacro()
1761{
1762// Write the analysis macro that will steer the analysis in grid mode.
1763 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1764 ofstream out;
1765 out.open(fAnalysisMacro.Data(), ios::out);
1766 if (!out.good()) {
1767 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1768 return;
1769 }
1770 TString func = fAnalysisMacro;
1771 TString type = "ESD";
1772 TString comment = "// Analysis using ";
1773 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1774 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1775 type = "AOD";
1776 comment += "AOD";
1777 }
0df6ccf2 1778 if (type!="AOD" && fFriendChainName!="") {
1779 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1780 return;
1781 }
c57f56b7 1782 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1783 else comment += " data";
1784 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1785 func.ReplaceAll(".C", "");
1786 out << "void " << func.Data() << "()" << endl;
1787 out << "{" << endl;
1788 out << comment.Data() << endl;
1789 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1790 out << " TStopwatch timer;" << endl;
1791 out << " timer.Start();" << endl << endl;
c57f56b7 1792 out << "// load base root libraries" << endl;
1793 out << " gSystem->Load(\"libTree\");" << endl;
1794 out << " gSystem->Load(\"libGeom\");" << endl;
1795 out << " gSystem->Load(\"libVMC\");" << endl;
1796 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 1797 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
57377eb5 1798 out << "// Load analysis framework libraries" << endl;
4e5c5506 1799 if (!fPackages) {
4e5c5506 1800 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1801 out << " gSystem->Load(\"libESD\");" << endl;
1802 out << " gSystem->Load(\"libAOD\");" << endl;
1803 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1804 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1805 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1806 } else {
4e5c5506 1807 TIter next(fPackages);
1808 TObject *obj;
57377eb5 1809 TString pkgname;
1810 Bool_t hasSTEERBase = kFALSE;
1811 Bool_t hasESD = kFALSE;
1812 Bool_t hasAOD = kFALSE;
1813 Bool_t hasANALYSIS = kFALSE;
1814 Bool_t hasANALYSISalice = kFALSE;
1815 Bool_t hasCORRFW = kFALSE;
1816 while ((obj=next())) {
1817 pkgname = obj->GetName();
4478e6f1 1818 if (pkgname == "STEERBase" ||
1819 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1820 if (pkgname == "ESD" ||
1821 pkgname == "ESD.par") hasESD = kTRUE;
1822 if (pkgname == "AOD" ||
1823 pkgname == "AOD.par") hasAOD = kTRUE;
1824 if (pkgname == "ANALYSIS" ||
1825 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1826 if (pkgname == "ANALYSISalice" ||
1827 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1828 if (pkgname == "CORRFW" ||
1829 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1830 }
1831 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1832 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1833 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1834 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1835 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1836 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1837 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1838 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1839 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1840 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1841 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1842 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1843 out << "// Compile other par packages" << endl;
1844 next.Reset();
fcc9bb6f 1845 while ((obj=next())) {
1846 pkgname = obj->GetName();
4478e6f1 1847 if (pkgname == "STEERBase" ||
1848 pkgname == "STEERBase.par" ||
1849 pkgname == "ESD" ||
1850 pkgname == "ESD.par" ||
1851 pkgname == "AOD" ||
1852 pkgname == "AOD.par" ||
1853 pkgname == "ANALYSIS" ||
1854 pkgname == "ANALYSIS.par" ||
1855 pkgname == "ANALYSISalice" ||
1856 pkgname == "ANALYSISalice.par" ||
1857 pkgname == "CORRFW" ||
1858 pkgname == "CORRFW.par") continue;
4e5c5506 1859 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1860 }
4e5c5506 1861 }
43da816a 1862 out << "// include path" << endl;
1863 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1864 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1865 if (fAdditionalLibs.Length()) {
1866 out << "// Add aditional AliRoot libraries" << endl;
1867 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1868 TIter next(list);
1869 TObjString *str;
1870 while((str=(TObjString*)next())) {
1871 if (str->GetString().Contains(".so"))
1872 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1873 }
1874 if (list) delete list;
1875 }
1876 out << endl;
c57f56b7 1877 out << "// analysis source to be compiled at runtime (if any)" << endl;
1878 if (fAnalysisSource.Length()) {
1879 TObjArray *list = fAnalysisSource.Tokenize(" ");
1880 TIter next(list);
1881 TObjString *str;
1882 while((str=(TObjString*)next())) {
1883 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1884 }
1885 if (list) delete list;
1886 }
1887 out << endl;
1888 out << "// connect to AliEn and make the chain" << endl;
1889 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1890 if (IsUsingTags()) {
1891 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1892 } else {
76535c98 1893 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1894 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1895 } else {
1896 out << " // Check if the macro to create the chain was provided" << endl;
1897 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1898 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1899 out << " return;" << endl;
1900 out << " }" << endl;
1901 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1902 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1903 }
c57f56b7 1904 }
1905 out << "// read the analysis manager from file" << endl;
f10e8481 1906 TString analysisFile = fExecutable;
1907 analysisFile.ReplaceAll(".sh", ".root");
1908 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1909 out << " if (!file) return;" << endl;
1910 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1911 out << " AliAnalysisManager *mgr = 0;" << endl;
1912 out << " TKey *key;" << endl;
1913 out << " while ((key=(TKey*)nextkey())) {" << endl;
1914 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1915 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1916 out << " };" << endl;
1917 out << " if (!mgr) {" << endl;
f10e8481 1918 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1919 out << " return;" << endl;
1920 out << " }" << endl << endl;
1921 out << " mgr->PrintStatus();" << endl;
1922 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1923 out << " timer.Stop();" << endl;
1924 out << " timer.Print();" << endl;
c57f56b7 1925 out << "}" << endl << endl;
1926 if (IsUsingTags()) {
1927 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1928 out << "{" << endl;
1929 out << "// Create a chain using tags from the xml file." << endl;
1930 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1931 out << " if (!coll) {" << endl;
1932 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1933 out << " return NULL;" << endl;
1934 out << " }" << endl;
1935 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1936 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1937 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1938 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1939 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1940 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1941 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1942 out << " // Check if the cuts configuration file was provided" << endl;
1943 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1944 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1945 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1946 out << " }" << endl;
0df6ccf2 1947 if (fFriendChainName=="") {
1948 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1949 } else {
1950 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1951 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1952 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1953 }
c57f56b7 1954 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1955 out << " chain->ls();" << endl;
1956 out << " return chain;" << endl;
fcc9bb6f 1957 out << "}" << endl << endl;
c57f56b7 1958 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1959 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1960 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1961 msg += " AliLHCTagCuts *lhcCuts,\n";
1962 msg += " AliDetectorTagCuts *detCuts,\n";
1963 msg += " AliEventTagCuts *evCuts)";
1964 Info("WriteAnalysisMacro", msg.Data());
1965 }
0df6ccf2 1966 }
1967 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1968 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1969 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1970 out << "{" << endl;
1971 out << "// Create a chain using url's from xml file" << endl;
1972 out << " TString treename = type;" << endl;
1973 out << " treename.ToLower();" << endl;
1974 out << " treename += \"Tree\";" << endl;
e02fee64 1975 out << " printf(\"***************************************\\n\");" << endl;
1976 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1977 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1978 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1979 out << " if (!coll) {" << endl;
1980 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1981 out << " return NULL;" << endl;
1982 out << " }" << endl;
1983 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1984 if(fFriendChainName!="") {
1985 out << " TChain *chainFriend = new TChain(treename);" << endl;
1986 }
c57f56b7 1987 out << " coll->Reset();" << endl;
0df6ccf2 1988 out << " while (coll->Next()) {" << endl;
1989 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1990 if(fFriendChainName!="") {
1991 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1992 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1993 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1994 out << " chainFriend->Add(fileFriend.Data());" << endl;
1995 }
1996 out << " }" << endl;
c57f56b7 1997 out << " if (!chain->GetNtrees()) {" << endl;
1998 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1999 out << " return NULL;" << endl;
2000 out << " }" << endl;
0df6ccf2 2001 if(fFriendChainName!="") {
2002 out << " chain->AddFriend(chainFriend);" << endl;
2003 }
c57f56b7 2004 out << " return chain;" << endl;
fcc9bb6f 2005 out << "}" << endl << endl;
c57f56b7 2006 }
4e5c5506 2007 if (fPackages) {
fcc9bb6f 2008 out <<"//________________________________________________________________________________" << endl;
4e5c5506 2009 out << "Bool_t SetupPar(const char *package) {" << endl;
2010 out << "// Compile the package and set it up." << endl;
2011 out << " TString pkgdir = package;" << endl;
2012 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 2013 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 2014 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2015 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2016 out << " // Check for BUILD.sh and execute" << endl;
2017 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2018 out << " printf(\"*******************************\\n\");" << endl;
2019 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2020 out << " printf(\"*******************************\\n\");" << endl;
2021 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 2022 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 2023 out << " gSystem->ChangeDirectory(cdir);" << endl;
2024 out << " return kFALSE;" << endl;
2025 out << " }" << endl;
2026 out << " } else {" << endl;
fcc9bb6f 2027 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 2028 out << " gSystem->ChangeDirectory(cdir);" << endl;
2029 out << " return kFALSE;" << endl;
2030 out << " }" << endl;
2031 out << " // Check for SETUP.C and execute" << endl;
2032 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2033 out << " printf(\"*******************************\\n\");" << endl;
2034 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2035 out << " printf(\"*******************************\\n\");" << endl;
2036 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2037 out << " } else {" << endl;
fcc9bb6f 2038 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 2039 out << " gSystem->ChangeDirectory(cdir);" << endl;
2040 out << " return kFALSE;" << endl;
2041 out << " }" << endl;
2042 out << " // Restore original workdir" << endl;
2043 out << " gSystem->ChangeDirectory(cdir);" << endl;
2044 out << " return kTRUE;" << endl;
2045 out << "}" << endl;
2046 }
c57f56b7 2047 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2048 }
2049 Bool_t copy = kTRUE;
2050 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2051 if (copy) {
2052 CdWork();
2053 TString workdir = gGrid->GetHomeDirectory();
2054 workdir += fGridWorkingDir;
2055 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2056 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2057 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2058 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2059 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2060 }
2061 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2062 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2063 }
2064}
2065
2066//______________________________________________________________________________
2067void AliAnalysisAlien::WriteExecutable()
2068{
2069// Generate the alien executable script.
2070 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2071 ofstream out;
2072 out.open(fExecutable.Data(), ios::out);
2073 if (out.bad()) {
5513444a 2074 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2075 return;
2076 }
2077 out << "#!/bin/bash" << endl;
c57f56b7 2078 out << "echo \"=========================================\"" << endl;
2079 out << "echo \"############## PATH : ##############\"" << endl;
2080 out << "echo $PATH" << endl;
2081 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2082 out << "echo $LD_LIBRARY_PATH" << endl;
2083 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2084 out << "echo $ROOTSYS" << endl;
2085 out << "echo \"############## which root : ##############\"" << endl;
2086 out << "which root" << endl;
2087 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2088 out << "echo $ALICE_ROOT" << endl;
2089 out << "echo \"############## which aliroot : ##############\"" << endl;
2090 out << "which aliroot" << endl;
9c5ddadc 2091 out << "echo \"############## system limits : ##############\"" << endl;
2092 out << "ulimit -a" << endl;
2093 out << "echo \"############## memory : ##############\"" << endl;
2094 out << "free -m" << endl;
c57f56b7 2095 out << "echo \"=========================================\"" << endl << endl;
74a53467 2096 // Make sure we can properly compile par files
2097 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2098 out << fExecutableCommand << " ";
631c0b05 2099 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2100 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2101 out << "echo \"############## memory after: ##############\"" << endl;
2102 out << "free -m" << endl;
2103 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2104 out << "dmesg | tail -n 10" << endl;
c57f56b7 2105 }
2106 Bool_t copy = kTRUE;
2107 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2108 if (copy) {
2109 CdWork();
2110 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2111 TString bindir = Form("%s/bin", workdir.Data());
2112 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2113 workdir += fGridWorkingDir;
2114 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2115 if (FileExists(executable)) gGrid->Rm(executable);
2116 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2117 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2118 }
2119}
2120
2121//______________________________________________________________________________
5513444a 2122void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2123{
2124// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2125// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2126// Next lines: full_path_to_dataset XXX (XXX is a string)
2127// To submit, one has to: submit jdl XXX for all lines
2128 ofstream out;
2129 out.open(filename, ios::out);
2130 if (out.bad()) {
2131 Error("WriteProductionFile", "Bad file name: %s", filename);
2132 return;
2133 }
2134 TString workdir = gGrid->GetHomeDirectory();
2135 workdir += fGridWorkingDir;
f5e8c702 2136 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2137 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2138 out << locjdl << " " << njobspermaster << endl;
5513444a 2139 Int_t nmasterjobs = fInputFiles->GetEntries();
2140 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2141 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2142 }
2143 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2144 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2145}
2146
2147//______________________________________________________________________________
c57f56b7 2148void AliAnalysisAlien::WriteValidationScript()
2149{
2150// Generate the alien validation script.
2151 // Generate the validation script
2152 TObjString *os;
0d5d317c 2153 TString validationScript = fExecutable;
2154 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2155 if (!Connect()) {
2156 Error("WriteValidationScript", "Alien connection required");
2157 return;
2158 }
2159 TString out_stream = "";
2160 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2161 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2162 ofstream out;
0d5d317c 2163 out.open(validationScript, ios::out);
c57f56b7 2164 out << "#!/bin/bash" << endl;
2165 out << "##################################################" << endl;
2166 out << "validateout=`dirname $0`" << endl;
2167 out << "validatetime=`date`" << endl;
2168 out << "validated=\"0\";" << endl;
2169 out << "error=0" << endl;
2170 out << "if [ -z $validateout ]" << endl;
2171 out << "then" << endl;
2172 out << " validateout=\".\"" << endl;
2173 out << "fi" << endl << endl;
2174 out << "cd $validateout;" << endl;
2175 out << "validateworkdir=`pwd`;" << endl << endl;
2176 out << "echo \"*******************************************************\"" << out_stream << endl;
2177 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2178 out << "" << endl;
2179 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2180 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2181 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2182 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2183 out << "ls -la ./" << out_stream << endl;
2184 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2185 out << "##################################################" << endl;
ebec370a 2186
2187 out << "" << endl;
2188 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2189 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2190 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2191 out << "" << endl;
2192
2193 out << "if [ ! -f stderr ] ; then" << endl;
2194 out << " error=1" << endl;
2195 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2196 out << " echo \"Error = $error\" " << out_stream << endl;
2197 out << "fi" << endl;
2198
2199 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2200 out << " error=1" << endl;
2201 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2202 out << " echo \"$parArch\" " << out_stream << endl;
2203 out << " echo \"Error = $error\" " << out_stream << endl;
2204 out << "fi" << endl;
2205
2206 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2207 out << " error=1" << endl;
2208 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2209 out << " echo \"$segViol\" " << out_stream << endl;
2210 out << " echo \"Error = $error\" " << out_stream << endl;
2211 out << "fi" << endl;
2212
2213 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2214 out << " error=1" << endl;
2215 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2216 out << " echo \"$segFault\" " << out_stream << endl;
2217 out << " echo \"Error = $error\" " << out_stream << endl;
2218 out << "fi" << endl;
2219
2220 // Part dedicated to the specific analyses running into the train
2221
c57f56b7 2222 TObjArray *arr = fOutputFiles.Tokenize(" ");
2223 TIter next1(arr);
2224 TString output_file;
2225 while ((os=(TObjString*)next1())) {
2226 output_file = os->GetString();
2227 Int_t index = output_file.Index("@");
2228 if (index > 0) output_file.Remove(index);
2229 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2230 out << " error=1" << endl;
2231 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2232 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2233 out << "fi" << endl;
2234 }
2235 delete arr;
923e2ca5 2236 out << "if ! [ -f outputs_valid ] ; then" << endl;
2237 out << " error=1" << endl;
2238 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2239 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2240 out << "fi" << endl;
2241
c57f56b7 2242 out << "if [ $error = 0 ] ; then" << endl;
2243 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2244 out << "fi" << endl;
2245
2246 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2247 out << "echo \"*******************************************************\"" << out_stream << endl;
2248 out << "cd -" << endl;
2249 out << "exit $error" << endl;
2250 }
2251 Bool_t copy = kTRUE;
2252 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2253 if (copy) {
2254 CdWork();
2255 TString workdir = gGrid->GetHomeDirectory();
2256 workdir += fGridWorkingDir;
0d5d317c 2257 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2258 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2259 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2260 }
2261}