]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update of iPhysicsSelection supporting histograms for all trigger types
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
cd11251e 57 fOutputToRunNo(0),
c57f56b7 58 fRunNumbers(),
59 fExecutable(),
0a1c1f7f 60 fExecutableCommand(),
c57f56b7 61 fArguments(),
631c0b05 62 fExecutableArgs(),
c57f56b7 63 fAnalysisMacro(),
64 fAnalysisSource(),
65 fAdditionalLibs(),
66 fSplitMode(),
67 fAPIVersion(),
68 fROOTVersion(),
69 fAliROOTVersion(),
648174cf 70 fExternalPackages(),
c57f56b7 71 fUser(),
72 fGridWorkingDir(),
73 fGridDataDir(),
74 fDataPattern(),
75 fGridOutputDir(),
76 fOutputArchive(),
77 fOutputFiles(),
78 fInputFormat(),
e7c71df0 79 fDatasetName(),
c57f56b7 80 fJDLName(),
bb885a9e 81 fMergeExcludes(),
f965131e 82 fIncludePath(),
bb885a9e 83 fCloseSE(),
0df6ccf2 84 fFriendChainName(),
c6cb3634 85 fJobTag(),
648174cf 86 fOutputSingle(),
5fce53f4 87 fRunPrefix(),
4e5c5506 88 fInputFiles(0),
89 fPackages(0)
c57f56b7 90{
91// Dummy ctor.
92 SetDefaults();
93}
94
95//______________________________________________________________________________
96AliAnalysisAlien::AliAnalysisAlien(const char *name)
97 :AliAnalysisGrid(name),
98 fGridJDL(NULL),
99 fPrice(0),
100 fTTL(0),
101 fSplitMaxInputFileNumber(0),
102 fMaxInitFailed(0),
103 fMasterResubmitThreshold(0),
bb885a9e 104 fNtestFiles(0),
319593fb 105 fNrunsPerMaster(0),
16a4353c 106 fMaxMergeFiles(0),
d2a409b2 107 fNsubmitted(0),
a3e84053 108 fProductionMode(0),
cd11251e 109 fOutputToRunNo(0),
c57f56b7 110 fRunNumbers(),
111 fExecutable(),
0a1c1f7f 112 fExecutableCommand(),
c57f56b7 113 fArguments(),
631c0b05 114 fExecutableArgs(),
c57f56b7 115 fAnalysisMacro(),
116 fAnalysisSource(),
117 fAdditionalLibs(),
118 fSplitMode(),
119 fAPIVersion(),
120 fROOTVersion(),
121 fAliROOTVersion(),
648174cf 122 fExternalPackages(),
c57f56b7 123 fUser(),
124 fGridWorkingDir(),
125 fGridDataDir(),
126 fDataPattern(),
127 fGridOutputDir(),
128 fOutputArchive(),
129 fOutputFiles(),
130 fInputFormat(),
e7c71df0 131 fDatasetName(),
c57f56b7 132 fJDLName(),
bb885a9e 133 fMergeExcludes(),
f965131e 134 fIncludePath(),
bb885a9e 135 fCloseSE(),
0df6ccf2 136 fFriendChainName(),
c6cb3634 137 fJobTag(),
648174cf 138 fOutputSingle(),
5fce53f4 139 fRunPrefix(),
4e5c5506 140 fInputFiles(0),
141 fPackages(0)
c57f56b7 142{
143// Default ctor.
144 SetDefaults();
145}
146
147//______________________________________________________________________________
148AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
149 :AliAnalysisGrid(other),
150 fGridJDL(NULL),
151 fPrice(other.fPrice),
152 fTTL(other.fTTL),
153 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
154 fMaxInitFailed(other.fMaxInitFailed),
155 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 156 fNtestFiles(other.fNtestFiles),
319593fb 157 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 158 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 159 fNsubmitted(other.fNsubmitted),
a3e84053 160 fProductionMode(other.fProductionMode),
cd11251e 161 fOutputToRunNo(other.fOutputToRunNo),
c57f56b7 162 fRunNumbers(other.fRunNumbers),
163 fExecutable(other.fExecutable),
0a1c1f7f 164 fExecutableCommand(other.fExecutableCommand),
c57f56b7 165 fArguments(other.fArguments),
631c0b05 166 fExecutableArgs(other.fExecutableArgs),
c57f56b7 167 fAnalysisMacro(other.fAnalysisMacro),
168 fAnalysisSource(other.fAnalysisSource),
169 fAdditionalLibs(other.fAdditionalLibs),
170 fSplitMode(other.fSplitMode),
171 fAPIVersion(other.fAPIVersion),
172 fROOTVersion(other.fROOTVersion),
173 fAliROOTVersion(other.fAliROOTVersion),
648174cf 174 fExternalPackages(other.fExternalPackages),
c57f56b7 175 fUser(other.fUser),
176 fGridWorkingDir(other.fGridWorkingDir),
177 fGridDataDir(other.fGridDataDir),
178 fDataPattern(other.fDataPattern),
179 fGridOutputDir(other.fGridOutputDir),
180 fOutputArchive(other.fOutputArchive),
181 fOutputFiles(other.fOutputFiles),
182 fInputFormat(other.fInputFormat),
e7c71df0 183 fDatasetName(other.fDatasetName),
c57f56b7 184 fJDLName(other.fJDLName),
bb885a9e 185 fMergeExcludes(other.fMergeExcludes),
f965131e 186 fIncludePath(other.fIncludePath),
bb885a9e 187 fCloseSE(other.fCloseSE),
0df6ccf2 188 fFriendChainName(other.fFriendChainName),
c6cb3634 189 fJobTag(other.fJobTag),
648174cf 190 fOutputSingle(other.fOutputSingle),
5fce53f4 191 fRunPrefix(other.fRunPrefix),
4e5c5506 192 fInputFiles(0),
193 fPackages(0)
c57f56b7 194{
195// Copy ctor.
196 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 197 fRunRange[0] = other.fRunRange[0];
198 fRunRange[1] = other.fRunRange[1];
c57f56b7 199 if (other.fInputFiles) {
200 fInputFiles = new TObjArray();
201 TIter next(other.fInputFiles);
202 TObject *obj;
203 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
204 fInputFiles->SetOwner();
205 }
4e5c5506 206 if (other.fPackages) {
207 fPackages = new TObjArray();
208 TIter next(other.fPackages);
209 TObject *obj;
210 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
211 fPackages->SetOwner();
212 }
c57f56b7 213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::~AliAnalysisAlien()
217{
218// Destructor.
219 if (fGridJDL) delete fGridJDL;
220 if (fInputFiles) delete fInputFiles;
4e5c5506 221 if (fPackages) delete fPackages;
c57f56b7 222}
223
224//______________________________________________________________________________
225AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
226{
227// Assignment.
228 if (this != &other) {
229 AliAnalysisGrid::operator=(other);
230 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
231 fPrice = other.fPrice;
232 fTTL = other.fTTL;
233 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
234 fMaxInitFailed = other.fMaxInitFailed;
235 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 236 fNtestFiles = other.fNtestFiles;
a3e84053 237 fNrunsPerMaster = other.fNrunsPerMaster;
238 fMaxMergeFiles = other.fMaxMergeFiles;
239 fNsubmitted = other.fNsubmitted;
240 fProductionMode = other.fProductionMode;
cd11251e 241 fOutputToRunNo = other.fOutputToRunNo;
c57f56b7 242 fRunNumbers = other.fRunNumbers;
243 fExecutable = other.fExecutable;
0a1c1f7f 244 fExecutableCommand = other.fExecutableCommand;
c57f56b7 245 fArguments = other.fArguments;
631c0b05 246 fExecutableArgs = other.fExecutableArgs;
c57f56b7 247 fAnalysisMacro = other.fAnalysisMacro;
248 fAnalysisSource = other.fAnalysisSource;
249 fAdditionalLibs = other.fAdditionalLibs;
250 fSplitMode = other.fSplitMode;
251 fAPIVersion = other.fAPIVersion;
252 fROOTVersion = other.fROOTVersion;
253 fAliROOTVersion = other.fAliROOTVersion;
648174cf 254 fExternalPackages = other.fExternalPackages;
c57f56b7 255 fUser = other.fUser;
256 fGridWorkingDir = other.fGridWorkingDir;
257 fGridDataDir = other.fGridDataDir;
258 fDataPattern = other.fDataPattern;
259 fGridOutputDir = other.fGridOutputDir;
260 fOutputArchive = other.fOutputArchive;
261 fOutputFiles = other.fOutputFiles;
262 fInputFormat = other.fInputFormat;
e7c71df0 263 fDatasetName = other.fDatasetName;
c57f56b7 264 fJDLName = other.fJDLName;
bb885a9e 265 fMergeExcludes = other.fMergeExcludes;
f965131e 266 fIncludePath = other.fIncludePath;
bb885a9e 267 fCloseSE = other.fCloseSE;
0df6ccf2 268 fFriendChainName = other.fFriendChainName;
c6cb3634 269 fJobTag = other.fJobTag;
648174cf 270 fOutputSingle = other.fOutputSingle;
5fce53f4 271 fRunPrefix = other.fRunPrefix;
c57f56b7 272 if (other.fInputFiles) {
273 fInputFiles = new TObjArray();
274 TIter next(other.fInputFiles);
275 TObject *obj;
276 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
277 fInputFiles->SetOwner();
278 }
4e5c5506 279 if (other.fPackages) {
280 fPackages = new TObjArray();
281 TIter next(other.fPackages);
282 TObject *obj;
283 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
284 fPackages->SetOwner();
285 }
c57f56b7 286 }
287 return *this;
288}
289
f965131e 290//______________________________________________________________________________
291void AliAnalysisAlien::AddIncludePath(const char *path)
292{
293// Add include path in the remote analysis macro.
294 TString p(path);
295 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
296 else fIncludePath += Form("-I%s ", path);
297}
298
c57f56b7 299//______________________________________________________________________________
300void AliAnalysisAlien::AddRunNumber(Int_t run)
301{
302// Add a run number to the list of runs to be processed.
303 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 304 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 305}
306
ee75cfc3 307//______________________________________________________________________________
308void AliAnalysisAlien::AddRunNumber(const char* run)
309{
310// Add a run number to the list of runs to be processed.
311 if (fRunNumbers.Length()) fRunNumbers += " ";
312 fRunNumbers += run;
313}
314
c57f56b7 315//______________________________________________________________________________
316void AliAnalysisAlien::AddDataFile(const char *lfn)
317{
318// Adds a data file to the input to be analysed. The file should be a valid LFN
319// or point to an existing file in the alien workdir.
320 if (!fInputFiles) fInputFiles = new TObjArray();
321 fInputFiles->Add(new TObjString(lfn));
322}
648174cf 323
324//______________________________________________________________________________
325void AliAnalysisAlien::AddExternalPackage(const char *package)
326{
327// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
328 if (fExternalPackages) fExternalPackages += " ";
329 fExternalPackages += package;
330}
331
c57f56b7 332//______________________________________________________________________________
333Bool_t AliAnalysisAlien::Connect()
334{
335// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
336 if (gGrid && gGrid->IsConnected()) return kTRUE;
337 if (!gSystem->Getenv("alien_API_USER")) {
338 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
339 gSystem->Getenv("UID"));
340 return kFALSE;
341 }
342 if (!gGrid) {
343 Info("Connect", "Trying to connect to AliEn ...");
344 TGrid::Connect("alien://");
345 }
346 if (!gGrid || !gGrid->IsConnected()) {
347 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
348 return kFALSE;
349 }
350 fUser = gGrid->GetUser();
351 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
352 return kTRUE;
353}
354
355//______________________________________________________________________________
356void AliAnalysisAlien::CdWork()
357{
358// Check validity of alien workspace. Create directory if possible.
359 if (!Connect()) {
360 Error("CdWork", "Alien connection required");
361 return;
362 }
363 TString homedir = gGrid->GetHomeDirectory();
364 TString workdir = homedir + fGridWorkingDir;
923e2ca5 365 if (DirectoryExists(workdir)) {
366 gGrid->Cd(workdir);
367 return;
368 }
369 // Work directory not existing - create it
370 gGrid->Cd(homedir);
371 if (gGrid->Mkdir(workdir)) {
372 gGrid->Cd(fGridWorkingDir);
373 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
374 } else {
375 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
376 workdir.Data(), homedir.Data());
377 fGridWorkingDir = "";
378 }
c57f56b7 379}
380
381//______________________________________________________________________________
382Bool_t AliAnalysisAlien::CheckInputData()
383{
384// Check validity of input data. If necessary, create xml files.
a8739e8a 385 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 386 if (!fGridDataDir.Length()) {
387 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
388 return kFALSE;
389 }
390 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
391 return kTRUE;
c57f56b7 392 }
393 // Process declared files
394 Bool_t is_collection = kFALSE;
395 Bool_t is_xml = kFALSE;
396 Bool_t use_tags = kFALSE;
397 Bool_t checked = kFALSE;
398 CdWork();
399 TString file;
400 TString workdir = gGrid->GetHomeDirectory();
401 workdir += fGridWorkingDir;
402 if (fInputFiles) {
403 TObjString *objstr;
404 TIter next(fInputFiles);
405 while ((objstr=(TObjString*)next())) {
406 file = workdir;
407 file += "/";
408 file += objstr->GetString();
409 // Store full lfn path
410 if (FileExists(file)) objstr->SetString(file);
411 else {
412 file = objstr->GetName();
413 if (!FileExists(objstr->GetName())) {
414 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
415 objstr->GetName(), workdir.Data());
416 return kFALSE;
417 }
418 }
419 Bool_t iscoll, isxml, usetags;
420 CheckDataType(file, iscoll, isxml, usetags);
421 if (!checked) {
422 checked = kTRUE;
423 is_collection = iscoll;
424 is_xml = isxml;
425 use_tags = usetags;
426 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
427 } else {
428 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
429 Error("CheckInputData", "Some conflict was found in the types of inputs");
430 return kFALSE;
431 }
432 }
433 }
434 }
435 // Process requested run numbers
a8739e8a 436 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 437 // Check validity of alien data directory
438 if (!fGridDataDir.Length()) {
439 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
440 return kFALSE;
441 }
923e2ca5 442 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 443 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
444 return kFALSE;
445 }
446 if (is_collection) {
447 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
448 return kFALSE;
449 }
450
451 if (checked && !is_xml) {
452 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
453 return kFALSE;
454 }
455 // Check validity of run number(s)
456 TObjArray *arr;
457 TObjString *os;
319593fb 458 Int_t nruns = 0;
904f9f5f 459 TString schunk, schunk2;
c57f56b7 460 TString path;
461 if (!checked) {
462 checked = kTRUE;
463 use_tags = fDataPattern.Contains("tag");
464 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
465 }
466 if (use_tags != fDataPattern.Contains("tag")) {
467 Error("CheckInputData", "Cannot mix input files using/not using tags");
468 return kFALSE;
469 }
470 if (fRunNumbers.Length()) {
a8739e8a 471 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 472 arr = fRunNumbers.Tokenize(" ");
473 TIter next(arr);
474 while ((os=(TObjString*)next())) {
475 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 476 if (!DirectoryExists(path)) {
477 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 478 continue;
c57f56b7 479 }
480 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
481 TString msg = "\n##### file: ";
482 msg += path;
483 msg += " type: xml_collection;";
484 if (use_tags) msg += " using_tags: Yes";
485 else msg += " using_tags: No";
486 Info("CheckDataType", msg.Data());
319593fb 487 if (fNrunsPerMaster<2) {
d2a409b2 488 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 489 } else {
490 nruns++;
491 if (((nruns-1)%fNrunsPerMaster) == 0) {
492 schunk = os->GetString();
493 }
494 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
495 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 496 AddDataFile(schunk);
319593fb 497 }
c57f56b7 498 }
499 delete arr;
a8739e8a 500 } else {
501 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
502 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 503 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 504 if (!DirectoryExists(path)) {
505// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 506 continue;
507 }
5fce53f4 508 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 509 TString msg = "\n##### file: ";
510 msg += path;
511 msg += " type: xml_collection;";
512 if (use_tags) msg += " using_tags: Yes";
513 else msg += " using_tags: No";
514 Info("CheckDataType", msg.Data());
319593fb 515 if (fNrunsPerMaster<2) {
5fce53f4 516 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 517 } else {
518 nruns++;
519 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 520 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 521 }
904f9f5f 522 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 523 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 524 schunk += schunk2;
d2a409b2 525 AddDataFile(schunk);
319593fb 526 }
a8739e8a 527 }
904f9f5f 528 if (!fInputFiles) {
529 schunk += schunk2;
530 AddDataFile(schunk);
531 }
c57f56b7 532 }
533 return kTRUE;
534}
535
536//______________________________________________________________________________
537Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
538{
539// Create dataset for the grid data directory + run number.
540 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
541 if (!Connect()) {
542 Error("CreateDataset", "Cannot create dataset with no grid connection");
543 return kFALSE;
544 }
545
546 // Cd workspace
547 CdWork();
548 TString workdir = gGrid->GetHomeDirectory();
549 workdir += fGridWorkingDir;
550
551 // Compose the 'find' command arguments
552 TString command;
553 TString options = "-x collection ";
bb885a9e 554 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 555 TString conditions = "";
556
557 TString file;
558 TString path;
319593fb 559 Int_t nruns = 0;
904f9f5f 560 TString schunk, schunk2;
ab254fd1 561 TGridCollection *cbase=0, *cadd=0;
d2a409b2 562 if (!fRunNumbers.Length() && !fRunRange[0]) {
563 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
564 // Make a single data collection from data directory.
565 path = fGridDataDir;
923e2ca5 566 if (!DirectoryExists(path)) {
d2a409b2 567 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
568 return kFALSE;
569 }
923e2ca5 570// CdWork();
d2a409b2 571 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
572 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 573 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 574 command = "find ";
575 command += options;
576 command += path;
577 command += " ";
578 command += pattern;
579 command += conditions;
84fcd93f 580 printf("command: %s\n", command.Data());
d2a409b2 581 TGridResult *res = gGrid->Command(command);
582 if (res) delete res;
583 // Write standard output to file
584 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
585 }
586 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
587 // Copy xml file to alien space
588 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
589 if (!FileExists(file)) {
590 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
591 return kFALSE;
592 }
593 // Update list of files to be processed.
594 }
595 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
596 return kTRUE;
597 }
c57f56b7 598 // Several runs
a8739e8a 599 if (fRunNumbers.Length()) {
600 TObjArray *arr = fRunNumbers.Tokenize(" ");
601 TObjString *os;
602 TIter next(arr);
603 while ((os=(TObjString*)next())) {
604 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 605 if (!DirectoryExists(path)) continue;
606// CdWork();
a8739e8a 607 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
608 else file = Form("%s.xml", os->GetString().Data());
319593fb 609 // If local collection file does not exist, create it via 'find' command.
610 if (gSystem->AccessPathName(file)) {
611 command = "find ";
612 command += options;
613 command += path;
614 command += pattern;
615 command += conditions;
616 TGridResult *res = gGrid->Command(command);
617 if (res) delete res;
618 // Write standard output to file
619 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
620 }
a8739e8a 621 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 622 // Check if there is one run per master job.
623 if (fNrunsPerMaster<2) {
624 if (FileExists(file)) {
625 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
626 continue;
627 }
628 // Copy xml file to alien space
629 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
630 if (!FileExists(file)) {
631 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
632 delete arr;
633 return kFALSE;
634 }
635 } else {
636 nruns++;
637 if (((nruns-1)%fNrunsPerMaster) == 0) {
638 schunk = os->GetString();
639 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
640 } else {
641 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 642 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 643 cbase->Add(cadd);
644 delete cadd;
645 }
646 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
647 continue;
648 }
649 schunk += Form("_%s.xml", os->GetString().Data());
650 if (FileExists(schunk)) {
651 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
652 continue;
653 }
84fcd93f 654 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 655 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 656 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 657 if (!FileExists(schunk)) {
658 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
659 delete arr;
660 return kFALSE;
661 }
662 }
a8739e8a 663 }
664 delete arr;
665 } else {
666 // Process a full run range.
667 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 668 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 669 if (!DirectoryExists(path)) continue;
670// CdWork();
a8739e8a 671 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 672 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
319593fb 673 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 674 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
675// gGrid->Rm(file);
676 continue;
677 }
319593fb 678 // If local collection file does not exist, create it via 'find' command.
679 if (gSystem->AccessPathName(file)) {
680 command = "find ";
681 command += options;
682 command += path;
683 command += pattern;
684 command += conditions;
685 TGridResult *res = gGrid->Command(command);
686 if (res) delete res;
687 // Write standard output to file
688 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
689 }
a8739e8a 690 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 691 // Check if there is one run per master job.
692 if (fNrunsPerMaster<2) {
693 if (FileExists(file)) {
694 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
695 continue;
696 }
697 // Copy xml file to alien space
698 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
699 if (!FileExists(file)) {
700 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
701 return kFALSE;
702 }
703 } else {
704 nruns++;
95e5b448 705 // Check if the collection for the chunk exist locally.
706 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
707 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 708 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 709 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 710 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 711 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
712 } else {
713 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
714 cbase->Add(cadd);
715 delete cadd;
716 }
904f9f5f 717 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
718 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 719 continue;
720 }
904f9f5f 721 schunk = schunk2;
319593fb 722 if (FileExists(schunk)) {
723 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
724 continue;
725 }
84fcd93f 726 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 727 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 728 if (FileExists(schunk)) {
729 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
730 continue;
731 }
319593fb 732 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
733 if (!FileExists(schunk)) {
734 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
735 return kFALSE;
736 }
737 }
c57f56b7 738 }
a8739e8a 739 }
c57f56b7 740 return kTRUE;
741}
742
743//______________________________________________________________________________
744Bool_t AliAnalysisAlien::CreateJDL()
745{
746// Generate a JDL file according to current settings. The name of the file is
747// specified by fJDLName.
748 Bool_t error = kFALSE;
749 TObjArray *arr = 0;
750 Bool_t copy = kTRUE;
751 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
752 Bool_t generate = kTRUE;
753 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
754 if (!Connect()) {
755 Error("CreateJDL", "Alien connection required");
756 return kFALSE;
757 }
758 // Check validity of alien workspace
759 CdWork();
760 TString workdir = gGrid->GetHomeDirectory();
761 workdir += fGridWorkingDir;
762 if (generate) {
763 TObjString *os;
764 if (!fInputFiles) {
765 Error("CreateJDL()", "Define some input files for your analysis.");
766 error = kTRUE;
767 }
768 // Compose list of input files
769 // Check if output files were defined
770 if (!fOutputFiles.Length()) {
771 Error("CreateJDL", "You must define at least one output file");
772 error = kTRUE;
773 }
774 // Check if an output directory was defined and valid
775 if (!fGridOutputDir.Length()) {
776 Error("CreateJDL", "You must define AliEn output directory");
777 error = kTRUE;
778 } else {
d2a409b2 779 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 780 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 781 if (gGrid->Mkdir(fGridOutputDir)) {
782 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
783 } else {
784 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 785 // error = kTRUE;
c57f56b7 786 }
787 }
788 gGrid->Cd(workdir);
789 }
790 // Exit if any error up to now
791 if (error) return kFALSE;
792 // Set JDL fields
793 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 794 fGridJDL->SetExecutable(fExecutable);
b5b9dee8 795 if (!fArguments.IsNull())
796 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
c57f56b7 797// fGridJDL->SetTTL((UInt_t)fTTL);
798 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
799 if (fMaxInitFailed > 0)
800 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
801 if (fSplitMaxInputFileNumber > 0)
802 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
803 if (fSplitMode.Length())
804 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
805// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
806 if (fAliROOTVersion.Length())
807 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
808 if (fROOTVersion.Length())
809 fGridJDL->AddToPackages("ROOT", fROOTVersion);
810 if (fAPIVersion.Length())
811 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
648174cf 812 if (!fExternalPackages.IsNull()) {
813 arr = fExternalPackages.Tokenize(" ");
814 TIter next(arr);
815 while ((os=(TObjString*)next())) {
816 TString pkgname = os->GetString();
817 Int_t index = pkgname.Index("::");
818 TString pkgversion = pkgname(index+2, pkgname.Length());
819 pkgname.Remove(index);
820 fGridJDL->AddToPackages(pkgname, pkgversion);
821 }
822 delete arr;
823 }
c57f56b7 824 fGridJDL->SetInputDataListFormat(fInputFormat);
825 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 826 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 827 TString analysisFile = fExecutable;
828 analysisFile.ReplaceAll(".sh", ".root");
829 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 830 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
831 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
832 if (fAdditionalLibs.Length()) {
833 arr = fAdditionalLibs.Tokenize(" ");
834 TIter next(arr);
835 while ((os=(TObjString*)next())) {
836 if (os->GetString().Contains(".so")) continue;
837 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
838 }
839 delete arr;
840 }
4e5c5506 841 if (fPackages) {
842 TIter next(fPackages);
843 TObject *obj;
844 while ((obj=next()))
845 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
846 }
c57f56b7 847 if (fOutputArchive.Length()) {
848 arr = fOutputArchive.Tokenize(" ");
849 TIter next(arr);
850 while ((os=(TObjString*)next()))
43da816a 851 if (!os->GetString().Contains("@") && fCloseSE.Length())
852 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
853 else
854 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 855 delete arr;
856 }
c57f56b7 857 arr = fOutputFiles.Tokenize(" ");
858 TIter next(arr);
43da816a 859 while ((os=(TObjString*)next())) {
e1eaf596 860 // Ignore ouputs in jdl that are also in outputarchive
861 TString sout = os->GetString();
862 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
863 if (fOutputArchive.Contains(sout)) continue;
43da816a 864 if (!os->GetString().Contains("@") && fCloseSE.Length())
865 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
866 else
867 fGridJDL->AddToOutputSandbox(os->GetString());
868 }
c57f56b7 869 delete arr;
870// fGridJDL->SetPrice((UInt_t)fPrice);
871 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 872 TString validationScript = fExecutable;
873 validationScript.ReplaceAll(".sh", "_validation.sh");
874 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 875 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 876 // Write a jdl with 2 input parameters: collection name and output dir name.
877 WriteJDL(copy);
c57f56b7 878 }
879 // Copy jdl to grid workspace
a8739e8a 880 if (copy) {
b5e4aaa7 881 // Check if an output directory was defined and valid
882 if (!fGridOutputDir.Length()) {
883 Error("CreateJDL", "You must define AliEn output directory");
884 return kFALSE;
885 } else {
886 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
887 if (!DirectoryExists(fGridOutputDir)) {
888 if (gGrid->Mkdir(fGridOutputDir)) {
889 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
890 } else {
891 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
892 return kFALSE;
893 }
894 }
895 gGrid->Cd(workdir);
896 }
648174cf 897 if (TestBit(AliAnalysisGrid::kSubmit)) {
898 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
899 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
900 if (fProductionMode)
901 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
902 if (FileExists(locjdl)) gGrid->Rm(locjdl);
903 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
904 }
c57f56b7 905 if (fAdditionalLibs.Length()) {
e7c71df0 906 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 907 TObjString *os;
908 TIter next(arr);
909 while ((os=(TObjString*)next())) {
c57f56b7 910 if (os->GetString().Contains(".so")) continue;
4e5c5506 911 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 912 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
913 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
914 }
915 delete arr;
916 }
4e5c5506 917 if (fPackages) {
918 TIter next(fPackages);
919 TObject *obj;
920 while ((obj=next())) {
921 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
922 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
923 }
924 }
c57f56b7 925 }
926 return kTRUE;
927}
928
a8739e8a 929//______________________________________________________________________________
d2a409b2 930Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 931{
932// Writes one or more JDL's corresponding to findex. If findex is negative,
933// all run numbers are considered in one go (jdl). For non-negative indices
934// they correspond to the indices in the array fInputFiles.
935 if (!fInputFiles) return kFALSE;
936 TObjString *os;
a8739e8a 937 TString workdir = gGrid->GetHomeDirectory();
938 workdir += fGridWorkingDir;
d2a409b2 939
940 if (!fRunNumbers.Length() && !fRunRange[0]) {
941 // One jdl with no parameters in case input data is specified by name.
a8739e8a 942 TIter next(fInputFiles);
943 while ((os=(TObjString*)next()))
944 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
648174cf 945 if (!fOutputSingle.IsNull())
946 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
947 else
948 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 949 } else {
d2a409b2 950 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
951 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
cd11251e 952 if (!fOutputSingle.IsNull()) {
953 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
954 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()));
955 } else {
956 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
957 }
a8739e8a 958 }
959
960
961 // Generate the JDL as a string
962 TString sjdl = fGridJDL->Generate();
963 Int_t index;
964 index = sjdl.Index("Executable");
965 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
966 index = sjdl.Index("Split ");
648174cf 967 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
a8739e8a 968 index = sjdl.Index("SplitMaxInputFileNumber");
969 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
970 index = sjdl.Index("InputDataCollection");
971 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 972 index = sjdl.Index("InputFile");
973 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
974 index = sjdl.Index("InputDataList ");
975 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
976 index = sjdl.Index("InputDataListFormat");
977 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
978 index = sjdl.Index("Price");
979 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
980 index = sjdl.Index("Requirements");
981 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
982 index = sjdl.Index("Packages");
983 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
984 index = sjdl.Index("User =");
985 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
986 index = sjdl.Index("TTL");
987 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
988 index = sjdl.Index("OutputFile");
989 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
990 index = sjdl.Index("OutputDir");
991 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
992 index = sjdl.Index("OutputArchive");
993 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
994 index = sjdl.Index("MaxInitFailed");
995 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
996 index = sjdl.Index("MasterResubmitThreshold");
997 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
998 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
999 index = sjdl.Index("Validationcommand");
1000 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
1001 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1002 sjdl.ReplaceAll("(member", "\n (member");
1003 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1004 sjdl.ReplaceAll("{", "{\n ");
1005 sjdl.ReplaceAll("};", "\n};");
1006 sjdl.ReplaceAll("{\n \n", "{\n");
1007 sjdl.ReplaceAll("\n\n", "\n");
1008 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
1009 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1010 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1011 index = sjdl.Index("JDLVariables");
1012 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
1013 // Write jdl to file
a8739e8a 1014 ofstream out;
d2a409b2 1015 out.open(fJDLName.Data(), ios::out);
a8739e8a 1016 if (out.bad()) {
d2a409b2 1017 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1018 return kFALSE;
1019 }
1020 out << sjdl << endl;
1021
1022 // Copy jdl to grid workspace
1023 if (!copy) {
1024 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1025 } else {
d2a409b2 1026 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1027 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 1028 if (fProductionMode)
b5fe9cba 1029 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 1030 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 1031 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 1032 }
1033 return kTRUE;
1034}
1035
c57f56b7 1036//______________________________________________________________________________
5513444a 1037Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1038{
1039// Returns true if file exists.
5513444a 1040 if (!gGrid) return kFALSE;
c57f56b7 1041 TGridResult *res = gGrid->Ls(lfn);
1042 if (!res) return kFALSE;
1043 TMap *map = dynamic_cast<TMap*>(res->At(0));
1044 if (!map) {
1045 delete res;
1046 return kFALSE;
1047 }
1048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1049 if (!objs || !objs->GetString().Length()) {
1050 delete res;
1051 return kFALSE;
1052 }
1053 delete res;
1054 return kTRUE;
1055}
1056
923e2ca5 1057//______________________________________________________________________________
1058Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1059{
1060// Returns true if directory exists. Can be also a path.
1061 if (!gGrid) return kFALSE;
1062 // Check if dirname is a path
1063 TString dirstripped = dirname;
1064 dirstripped = dirstripped.Strip();
1065 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1066 TString dir = gSystem->BaseName(dirstripped);
1067 dir += "/";
1068 TString path = gSystem->DirName(dirstripped);
1069 TGridResult *res = gGrid->Ls(path, "-F");
1070 if (!res) return kFALSE;
1071 TIter next(res);
1072 TMap *map;
1073 TObject *obj;
1074 while ((map=dynamic_cast<TMap*>(next()))) {
1075 obj = map->GetValue("name");
1076 if (!obj) break;
1077 if (dir == obj->GetName()) {
1078 delete res;
1079 return kTRUE;
1080 }
1081 }
1082 delete res;
1083 return kFALSE;
1084}
1085
c57f56b7 1086//______________________________________________________________________________
1087void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1088{
1089// Check input data type.
1090 is_collection = kFALSE;
1091 is_xml = kFALSE;
1092 use_tags = kFALSE;
1093 if (!gGrid) {
1094 Error("CheckDataType", "No connection to grid");
1095 return;
1096 }
1097 is_collection = IsCollection(lfn);
1098 TString msg = "\n##### file: ";
1099 msg += lfn;
1100 if (is_collection) {
1101 msg += " type: raw_collection;";
1102 // special treatment for collections
1103 is_xml = kFALSE;
1104 // check for tag files in the collection
1105 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1106 if (!res) {
1107 msg += " using_tags: No (unknown)";
1108 Info("CheckDataType", msg.Data());
1109 return;
1110 }
1111 const char* typeStr = res->GetKey(0, "origLFN");
1112 if (!typeStr || !strlen(typeStr)) {
1113 msg += " using_tags: No (unknown)";
1114 Info("CheckDataType", msg.Data());
1115 return;
1116 }
1117 TString file = typeStr;
1118 use_tags = file.Contains(".tag");
1119 if (use_tags) msg += " using_tags: Yes";
1120 else msg += " using_tags: No";
1121 Info("CheckDataType", msg.Data());
1122 return;
1123 }
1124 TString slfn(lfn);
1125 slfn.ToLower();
1126 is_xml = slfn.Contains(".xml");
1127 if (is_xml) {
1128 // Open xml collection and check if there are tag files inside
1129 msg += " type: xml_collection;";
1130 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1131 if (!coll) {
1132 msg += " using_tags: No (unknown)";
1133 Info("CheckDataType", msg.Data());
1134 return;
1135 }
1136 TMap *map = coll->Next();
1137 if (!map) {
1138 msg += " using_tags: No (unknown)";
1139 Info("CheckDataType", msg.Data());
1140 return;
1141 }
1142 map = (TMap*)map->GetValue("");
1143 TString file;
1144 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1145 use_tags = file.Contains(".tag");
1146 delete coll;
1147 if (use_tags) msg += " using_tags: Yes";
1148 else msg += " using_tags: No";
1149 Info("CheckDataType", msg.Data());
1150 return;
1151 }
1152 use_tags = slfn.Contains(".tag");
1153 if (slfn.Contains(".root")) msg += " type: root file;";
1154 else msg += " type: unhnown file;";
1155 if (use_tags) msg += " using_tags: Yes";
1156 else msg += " using_tags: No";
1157 Info("CheckDataType", msg.Data());
1158}
1159
4e5c5506 1160//______________________________________________________________________________
1161void AliAnalysisAlien::EnablePackage(const char *package)
1162{
1163// Enables a par file supposed to exist in the current directory.
1164 TString pkg(package);
1165 pkg.ReplaceAll(".par", "");
1166 pkg += ".par";
1167 if (gSystem->AccessPathName(pkg)) {
1168 Error("EnablePackage", "Package %s not found", pkg.Data());
1169 return;
1170 }
1171 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1172 Info("EnablePackage", "AliEn plugin will use .par packages");
1173 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1174 if (!fPackages) {
1175 fPackages = new TObjArray();
1176 fPackages->SetOwner();
1177 }
1178 fPackages->Add(new TObjString(pkg));
1179}
1180
d2a409b2 1181//______________________________________________________________________________
1182const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1183{
1184// Get job status for all jobs with jobid>jobidstart.
1185 static char mstatus[20];
1186 mstatus[0] = '\0';
1187 nrunning = 0;
1188 nwaiting = 0;
1189 nerror = 0;
1190 ndone = 0;
1191 TGridJobStatusList *list = gGrid->Ps("");
1192 if (!list) return mstatus;
1193 Int_t nentries = list->GetSize();
1194 TGridJobStatus *status;
1195 Int_t pid;
1196 for (Int_t ijob=0; ijob<nentries; ijob++) {
1197 status = (TGridJobStatus *)list->At(ijob);
1198 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1199 if (pid<jobidstart) continue;
1200 if (pid == lastid) {
1201 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1202 }
1203 switch (status->GetStatus()) {
1204 case TGridJobStatus::kWAITING:
1205 nwaiting++; break;
1206 case TGridJobStatus::kRUNNING:
1207 nrunning++; break;
1208 case TGridJobStatus::kABORTED:
1209 case TGridJobStatus::kFAIL:
1210 case TGridJobStatus::kUNKNOWN:
1211 nerror++; break;
1212 case TGridJobStatus::kDONE:
1213 ndone++;
1214 }
1215 }
1216 list->Delete();
1217 delete list;
1218 return mstatus;
1219}
1220
c57f56b7 1221//______________________________________________________________________________
1222Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1223{
1224// Returns true if file is a collection. Functionality duplicated from
1225// TAlien::Type() because we don't want to directly depend on TAlien.
1226 if (!gGrid) {
1227 Error("IsCollection", "No connection to grid");
1228 return kFALSE;
1229 }
1230 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1231 if (!res) return kFALSE;
1232 const char* typeStr = res->GetKey(0, "type");
1233 if (!typeStr || !strlen(typeStr)) return kFALSE;
1234 if (!strcmp(typeStr, "collection")) return kTRUE;
1235 delete res;
1236 return kFALSE;
1237}
1238
16a4353c 1239//______________________________________________________________________________
1240void AliAnalysisAlien::Print(Option_t *) const
1241{
1242// Print current plugin settings.
84fcd93f 1243 printf("### AliEn analysis plugin current settings ###\n");
1244 printf("= Production mode:______________________________ %d\n", fProductionMode);
1245 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1246 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1247 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1248 if (fUser.Length())
84fcd93f 1249 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1250 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1251 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1252 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1253 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1254 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1255 if (fRunNumbers.Length())
84fcd93f 1256 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1257 if (fRunRange[0])
5fce53f4 1258 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1259 if (!fRunRange[0] && !fRunNumbers.Length()) {
1260 TIter next(fInputFiles);
1261 TObject *obj;
1262 TString list;
1263 while ((obj=next())) list += obj->GetName();
84fcd93f 1264 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1265 }
1266 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1267 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1268 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1269 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1270 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1271 printf("=====================================================================\n");
1272 printf("= Job price: ___________________________________ %d\n", fPrice);
1273 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1274 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1275 if (fMaxInitFailed>0)
84fcd93f 1276 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1277 if (fMasterResubmitThreshold>0)
84fcd93f 1278 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1279 if (fNrunsPerMaster>0)
84fcd93f 1280 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1281 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1282 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1283 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1284 if (fArguments.Length())
84fcd93f 1285 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1286 if (fExecutableArgs.Length())
1287 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1288 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1289 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1290 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1291 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1292 if (fDatasetName)
84fcd93f 1293 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1294 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1295 if (fIncludePath.Data())
84fcd93f 1296 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1297 if (fCloseSE.Length())
84fcd93f 1298 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1299 if (fFriendChainName.Length())
84fcd93f 1300 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1301 if (fPackages) {
1302 TIter next(fPackages);
1303 TObject *obj;
1304 TString list;
1305 while ((obj=next())) list += obj->GetName();
84fcd93f 1306 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1307 }
1308}
1309
c57f56b7 1310//______________________________________________________________________________
1311void AliAnalysisAlien::SetDefaults()
1312{
1313// Set default values for everything. What cannot be filled will be left empty.
1314 if (fGridJDL) delete fGridJDL;
1315 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1316 fPrice = 1;
1317 fTTL = 30000;
1318 fSplitMaxInputFileNumber = 100;
1319 fMaxInitFailed = 0;
1320 fMasterResubmitThreshold = 0;
bb885a9e 1321 fNtestFiles = 10;
a8739e8a 1322 fRunRange[0] = 0;
1323 fRunRange[1] = 0;
319593fb 1324 fNrunsPerMaster = 1;
16a4353c 1325 fMaxMergeFiles = 100;
c57f56b7 1326 fRunNumbers = "";
1327 fExecutable = "analysis.sh";
0a1c1f7f 1328 fExecutableCommand = "root -b -q";
c57f56b7 1329 fArguments = "";
631c0b05 1330 fExecutableArgs = "";
c57f56b7 1331 fAnalysisMacro = "myAnalysis.C";
1332 fAnalysisSource = "";
1333 fAdditionalLibs = "";
1334 fSplitMode = "se";
1335 fAPIVersion = "";
1336 fROOTVersion = "";
1337 fAliROOTVersion = "";
1338 fUser = ""; // Your alien user name
1339 fGridWorkingDir = "";
1340 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1341 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1342 fFriendChainName = "";
c57f56b7 1343 fGridOutputDir = "output";
1344 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1345 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1346 fInputFormat = "xml-single";
1347 fJDLName = "analysis.jdl";
c6cb3634 1348 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1349 fMergeExcludes = "";
c57f56b7 1350}
1351
1352//______________________________________________________________________________
1353Bool_t AliAnalysisAlien::MergeOutputs()
1354{
1355// Merge analysis outputs existing in the AliEn space.
1356 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1357 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1358 if (!Connect()) {
1359 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1360 return kFALSE;
1361 }
1362 // Get the output path
d2a409b2 1363 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1364 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1365 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1366 return kFALSE;
1367 }
1368 if (!fOutputFiles.Length()) {
1369 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1370 return kFALSE;
1371 }
1372 TObjArray *list = fOutputFiles.Tokenize(" ");
1373 TIter next(list);
1374 TObjString *str;
1375 TString command;
1376 TString output_file;
16a4353c 1377 TString output_chunk;
1378 TString previous_chunk;
1379 Int_t count_chunk = 0;
1380 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1381 Bool_t merged = kTRUE;
1382 while((str=(TObjString*)next())) {
1383 output_file = str->GetString();
1384 Int_t index = output_file.Index("@");
1385 if (index > 0) output_file.Remove(index);
319593fb 1386 // Skip already merged outputs
1387 if (!gSystem->AccessPathName(output_file)) {
1388 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1389 continue;
1390 }
bb885a9e 1391 if (fMergeExcludes.Length() &&
1392 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1393 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1394 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1395 printf("command: %s\n", command.Data());
c57f56b7 1396 TGridResult *res = gGrid->Command(command);
1397 if (!res) continue;
1398 TFileMerger *fm = 0;
1399 TIter nextmap(res);
ab254fd1 1400 TMap *map = 0;
16a4353c 1401 previous_chunk = "";
1402 count_chunk = 0;
319593fb 1403 // Check if there is a merge operation to resume
1404 output_chunk = output_file;
1405 output_chunk.ReplaceAll(".root", "_*.root");
1406 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1407 while (1) {
1408 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1409 if (!map) {
1410 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1411 delete res;
1412 return kFALSE;
1413 }
1414 output_chunk = output_file;
1415 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1416 printf("%s\n", output_chunk.Data());
319593fb 1417 count_chunk++;
1418 if (gSystem->AccessPathName(output_chunk)) continue;
1419 // Merged file with chunks up to <count_chunk> found
84fcd93f 1420 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1421 previous_chunk = output_chunk;
1422 break;
1423 }
1424 }
16a4353c 1425 count_zero = fMaxMergeFiles;
c57f56b7 1426 while ((map=(TMap*)nextmap())) {
16a4353c 1427 // Loop 'find' results and get next LFN
1428 if (count_zero == fMaxMergeFiles) {
1429 // First file in chunk - create file merger and add previous chunk if any.
1430 fm = new TFileMerger(kFALSE);
1431 fm->SetFastMethod(kTRUE);
1432 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1433 output_chunk = output_file;
1434 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1435 }
1436 // If last file found, put merged results in the output file
1437 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1438 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1439 if (!objs || !objs->GetString().Length()) {
16a4353c 1440 // Nothing found - skip this output
c57f56b7 1441 delete res;
16a4353c 1442 delete fm;
1443 break;
1444 }
1445 // Add file to be merged and decrement chunk counter.
1446 fm->AddFile(objs->GetString());
1447 count_zero--;
1448 if (count_zero==0 || map == res->Last()) {
1449 fm->OutputFile(output_chunk);
1450 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1451 // Nothing found - skip this output
1452 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1453 delete res;
1454 delete fm;
1455 break;
1456 }
1457 // Merge the outputs, then go to next chunk
1458 if (!fm->Merge()) {
1459 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1460 delete res;
1461 delete fm;
1462 merged = kFALSE;
1463 break;
1464 } else {
1465 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1466 gSystem->Unlink(previous_chunk);
16a4353c 1467 }
1468 if (map == res->Last()) {
1469 delete res;
1470 delete fm;
1471 break;
1472 }
1473 count_chunk++;
1474 count_zero = fMaxMergeFiles;
1475 previous_chunk = output_chunk;
c57f56b7 1476 }
c57f56b7 1477 }
c57f56b7 1478 }
1479 if (!merged) {
1480 Error("MergeOutputs", "Terminate() will NOT be executed");
1481 }
1482 return merged;
1483}
1484
bb885a9e 1485//______________________________________________________________________________
1486void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1487{
1488// Use the output files connected to output containers from the analysis manager
1489// rather than the files defined by SetOutputFiles
1490 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1491 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1492 analysis manager");
1493 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1494}
1495
5513444a 1496//______________________________________________________________________________
1497Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1498{
1499// Start remote grid analysis.
1500
43da816a 1501 // Check if output files have to be taken from the analysis manager
1502 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1503 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1504 if (!mgr || !mgr->IsInitialized()) {
1505 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1506 return kFALSE;
43da816a 1507 }
1508 fOutputFiles = "";
1509 TIter next(mgr->GetOutputs());
1510 AliAnalysisDataContainer *output;
1511 while ((output=(AliAnalysisDataContainer*)next())) {
1512 const char *filename = output->GetFileName();
1513 if (!(strcmp(filename, "default"))) {
1514 if (!mgr->GetOutputEventHandler()) continue;
1515 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1516 }
84fcd93f 1517 if (fOutputFiles.Contains(filename)) continue;
43da816a 1518 if (fOutputFiles.Length()) fOutputFiles += " ";
1519 fOutputFiles += filename;
1520 }
c07b9ce2 1521 // Add extra files registered to the analysis manager
1522 if (mgr->GetExtraFiles().Length()) {
1523 if (fOutputFiles.Length()) fOutputFiles += " ";
1524 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1525 }
43da816a 1526 }
f7b1cbc2 1527// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1528 if (TestBit(AliAnalysisGrid::kOffline)) {
1529 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1530 \n there nor any job run. You can revise the JDL and analysis \
1531 \n macro then run the same in \"submit\" mode.");
1532 } else if (TestBit(AliAnalysisGrid::kTest)) {
1533 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1534 \n dataset.");
1535 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1536 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1537 \n space and job submitted.");
1538 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1539 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1540 return kTRUE;
c57f56b7 1541 } else {
1542 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1543 }
1544
1545 if (!Connect()) {
1546 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1547 return kFALSE;
16a4353c 1548 }
1549 Print();
c57f56b7 1550 if (!CheckInputData()) {
1551 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1552 return kFALSE;
c57f56b7 1553 }
1554 CreateDataset(fDataPattern);
1555 WriteAnalysisFile();
1556 WriteAnalysisMacro();
1557 WriteExecutable();
1558 WriteValidationScript();
5513444a 1559 if (!CreateJDL()) return kFALSE;
1560 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1561 if (TestBit(AliAnalysisGrid::kTest)) {
1562 // Locally testing the analysis
1563 Info("StartAnalysis", "\n_______________________________________________________________________ \
1564 \n Running analysis script in a daughter shell as on a worker node \
1565 \n_______________________________________________________________________");
1566 TObjArray *list = fOutputFiles.Tokenize(" ");
1567 TIter next(list);
1568 TObjString *str;
1569 TString output_file;
1570 while((str=(TObjString*)next())) {
1571 output_file = str->GetString();
1572 Int_t index = output_file.Index("@");
1573 if (index > 0) output_file.Remove(index);
43da816a 1574 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1575 }
1576 delete list;
1577 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1578 TString validationScript = fExecutable;
1579 validationScript.ReplaceAll(".sh", "_validation.sh");
1580 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1581// gSystem->Exec("cat stdout");
5513444a 1582 return kFALSE;
c57f56b7 1583 }
5513444a 1584 // Check if submitting is managed by LPM manager
a3e84053 1585 if (fProductionMode) {
5513444a 1586 TString prodfile = fJDLName;
1587 prodfile.ReplaceAll(".jdl", ".prod");
1588 WriteProductionFile(prodfile);
1589 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1590 return kFALSE;
1591 }
a8739e8a 1592 // Submit AliEn job(s)
d2a409b2 1593 gGrid->Cd(fGridOutputDir);
a8739e8a 1594 TGridResult *res;
c57f56b7 1595 TString jobID = "";
d2a409b2 1596 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1597 // Submit a given xml or a set of runs
a8739e8a 1598 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1599 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1600 if (res) {
1601 const char *cjobId = res->GetKey(0,"jobId");
1602 if (!cjobId) {
1603 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1604 return kFALSE;
a8739e8a 1605 } else {
1606 Info("StartAnalysis", "\n_______________________________________________________________________ \
1607 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1608 \n_______________________________________________________________________",
1609 fJDLName.Data(), cjobId);
1610 jobID = cjobId;
1611 }
1612 delete res;
1613 }
1614 } else {
d2a409b2 1615 // Submit for a range of enumeration of runs.
1616 Submit();
c57f56b7 1617 }
a8739e8a 1618
c57f56b7 1619 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1620 \n You may exit at any time and terminate the job later using the option <terminate> \
1621 \n ##################################################################################", jobID.Data());
bb885a9e 1622 gSystem->Exec("aliensh");
5513444a 1623 return kTRUE;
c57f56b7 1624}
1625
d2a409b2 1626//______________________________________________________________________________
1627void AliAnalysisAlien::Submit()
1628{
1629// Submit all master jobs.
1630 Int_t nmasterjobs = fInputFiles->GetEntries();
1631 Long_t tshoot = gSystem->Now();
1632 if (!fNsubmitted) SubmitNext();
1633 while (fNsubmitted < nmasterjobs) {
1634 Long_t now = gSystem->Now();
1635 if ((now-tshoot)>30000) {
1636 tshoot = now;
1637 SubmitNext();
1638 }
1639 }
1640}
1641
1642//______________________________________________________________________________
1643void AliAnalysisAlien::SubmitNext()
1644{
1645// Submit next bunch of master jobs if the queue is free.
1646 static Bool_t iscalled = kFALSE;
1647 static Int_t firstmaster = 0;
1648 static Int_t lastmaster = 0;
1649 static Int_t npermaster = 0;
1650 if (iscalled) return;
1651 iscalled = kTRUE;
1652 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1653 Int_t ntosubmit = 0;
1654 TGridResult *res;
1655 TString jobID = "";
1656 if (!fNsubmitted) ntosubmit = 1;
1657 else {
1658 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1659 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1660 // If last master not split, just return
1661 if (status != "SPLIT") {iscalled = kFALSE; return;}
1662 // No more than 100 waiting jobs
1663 if (nwaiting>100) {iscalled = kFALSE; return;}
1664 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1665 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1666 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1667 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1668 }
1669 Int_t nmasterjobs = fInputFiles->GetEntries();
1670 for (Int_t i=0; i<ntosubmit; i++) {
1671 // Submit for a range of enumeration of runs.
1672 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1673 TString query;
cd11251e 1674 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
1675 runOutDir.ReplaceAll(".xml", "");
1676 if (fOutputToRunNo)
1677 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
1678 else
1679 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1680 printf("********* %s\n",query.Data());
d2a409b2 1681 res = gGrid->Command(query);
1682 if (res) {
98ca124f 1683 TString cjobId1 = res->GetKey(0,"jobId");
1684 if (!cjobId1.Length()) {
d2a409b2 1685 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1686 iscalled = kFALSE;
1687 return;
1688 } else {
1689 Info("StartAnalysis", "\n_______________________________________________________________________ \
1690 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1691 \n_______________________________________________________________________",
98ca124f 1692 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1693 jobID += cjobId1;
1694 jobID += " ";
98ca124f 1695 lastmaster = cjobId1.Atoi();
d2a409b2 1696 if (!firstmaster) firstmaster = lastmaster;
1697 fNsubmitted++;
1698 }
1699 delete res;
1700 }
1701 }
1702 iscalled = kFALSE;
1703}
1704
c57f56b7 1705//______________________________________________________________________________
1706void AliAnalysisAlien::WriteAnalysisFile()
1707{
f10e8481 1708// Write current analysis manager into the file <analysisFile>
1709 TString analysisFile = fExecutable;
1710 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1711 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1712 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1713 if (!mgr || !mgr->IsInitialized()) {
1714 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1715 return;
1716 }
1717 // Check analysis type
1718 TObject *handler;
1719 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1720 handler = (TObject*)mgr->GetInputEventHandler();
1721 if (handler) {
1722 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1723 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1724 }
1725 TDirectory *cdir = gDirectory;
f10e8481 1726 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1727 if (file) {
1728 mgr->Write();
1729 delete file;
1730 }
1731 if (cdir) cdir->cd();
f10e8481 1732 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1733 }
1734 Bool_t copy = kTRUE;
1735 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1736 if (copy) {
1737 CdWork();
1738 TString workdir = gGrid->GetHomeDirectory();
1739 workdir += fGridWorkingDir;
f10e8481 1740 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1741 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1742 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1743 }
1744}
1745
1746//______________________________________________________________________________
1747void AliAnalysisAlien::WriteAnalysisMacro()
1748{
1749// Write the analysis macro that will steer the analysis in grid mode.
1750 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1751 ofstream out;
1752 out.open(fAnalysisMacro.Data(), ios::out);
1753 if (!out.good()) {
1754 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1755 return;
1756 }
1757 TString func = fAnalysisMacro;
1758 TString type = "ESD";
1759 TString comment = "// Analysis using ";
1760 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1761 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1762 type = "AOD";
1763 comment += "AOD";
1764 }
0df6ccf2 1765 if (type!="AOD" && fFriendChainName!="") {
1766 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1767 return;
1768 }
c57f56b7 1769 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1770 else comment += " data";
1771 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1772 func.ReplaceAll(".C", "");
1773 out << "void " << func.Data() << "()" << endl;
1774 out << "{" << endl;
1775 out << comment.Data() << endl;
1776 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1777 out << " TStopwatch timer;" << endl;
1778 out << " timer.Start();" << endl << endl;
c57f56b7 1779 out << "// load base root libraries" << endl;
1780 out << " gSystem->Load(\"libTree\");" << endl;
1781 out << " gSystem->Load(\"libGeom\");" << endl;
1782 out << " gSystem->Load(\"libVMC\");" << endl;
1783 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 1784 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
57377eb5 1785 out << "// Load analysis framework libraries" << endl;
4e5c5506 1786 if (!fPackages) {
4e5c5506 1787 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1788 out << " gSystem->Load(\"libESD\");" << endl;
1789 out << " gSystem->Load(\"libAOD\");" << endl;
1790 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1791 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1792 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1793 } else {
4e5c5506 1794 TIter next(fPackages);
1795 TObject *obj;
57377eb5 1796 TString pkgname;
1797 Bool_t hasSTEERBase = kFALSE;
1798 Bool_t hasESD = kFALSE;
1799 Bool_t hasAOD = kFALSE;
1800 Bool_t hasANALYSIS = kFALSE;
1801 Bool_t hasANALYSISalice = kFALSE;
1802 Bool_t hasCORRFW = kFALSE;
1803 while ((obj=next())) {
1804 pkgname = obj->GetName();
4478e6f1 1805 if (pkgname == "STEERBase" ||
1806 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1807 if (pkgname == "ESD" ||
1808 pkgname == "ESD.par") hasESD = kTRUE;
1809 if (pkgname == "AOD" ||
1810 pkgname == "AOD.par") hasAOD = kTRUE;
1811 if (pkgname == "ANALYSIS" ||
1812 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1813 if (pkgname == "ANALYSISalice" ||
1814 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1815 if (pkgname == "CORRFW" ||
1816 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1817 }
1818 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1819 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1820 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1821 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1822 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1823 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1824 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1825 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1826 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1827 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1828 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1829 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1830 out << "// Compile other par packages" << endl;
1831 next.Reset();
fcc9bb6f 1832 while ((obj=next())) {
1833 pkgname = obj->GetName();
4478e6f1 1834 if (pkgname == "STEERBase" ||
1835 pkgname == "STEERBase.par" ||
1836 pkgname == "ESD" ||
1837 pkgname == "ESD.par" ||
1838 pkgname == "AOD" ||
1839 pkgname == "AOD.par" ||
1840 pkgname == "ANALYSIS" ||
1841 pkgname == "ANALYSIS.par" ||
1842 pkgname == "ANALYSISalice" ||
1843 pkgname == "ANALYSISalice.par" ||
1844 pkgname == "CORRFW" ||
1845 pkgname == "CORRFW.par") continue;
4e5c5506 1846 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1847 }
4e5c5506 1848 }
43da816a 1849 out << "// include path" << endl;
1850 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1851 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1852 if (fAdditionalLibs.Length()) {
1853 out << "// Add aditional AliRoot libraries" << endl;
1854 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1855 TIter next(list);
1856 TObjString *str;
1857 while((str=(TObjString*)next())) {
1858 if (str->GetString().Contains(".so"))
1859 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1860 }
1861 if (list) delete list;
1862 }
1863 out << endl;
c57f56b7 1864 out << "// analysis source to be compiled at runtime (if any)" << endl;
1865 if (fAnalysisSource.Length()) {
1866 TObjArray *list = fAnalysisSource.Tokenize(" ");
1867 TIter next(list);
1868 TObjString *str;
1869 while((str=(TObjString*)next())) {
1870 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1871 }
1872 if (list) delete list;
1873 }
1874 out << endl;
1875 out << "// connect to AliEn and make the chain" << endl;
1876 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1877 if (IsUsingTags()) {
1878 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1879 } else {
76535c98 1880 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1881 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1882 } else {
1883 out << " // Check if the macro to create the chain was provided" << endl;
1884 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1885 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1886 out << " return;" << endl;
1887 out << " }" << endl;
1888 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1889 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1890 }
c57f56b7 1891 }
1892 out << "// read the analysis manager from file" << endl;
f10e8481 1893 TString analysisFile = fExecutable;
1894 analysisFile.ReplaceAll(".sh", ".root");
1895 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1896 out << " if (!file) return;" << endl;
1897 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1898 out << " AliAnalysisManager *mgr = 0;" << endl;
1899 out << " TKey *key;" << endl;
1900 out << " while ((key=(TKey*)nextkey())) {" << endl;
1901 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1902 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1903 out << " };" << endl;
1904 out << " if (!mgr) {" << endl;
f10e8481 1905 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1906 out << " return;" << endl;
1907 out << " }" << endl << endl;
1908 out << " mgr->PrintStatus();" << endl;
1909 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1910 out << " timer.Stop();" << endl;
1911 out << " timer.Print();" << endl;
c57f56b7 1912 out << "}" << endl << endl;
1913 if (IsUsingTags()) {
1914 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1915 out << "{" << endl;
1916 out << "// Create a chain using tags from the xml file." << endl;
1917 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1918 out << " if (!coll) {" << endl;
1919 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1920 out << " return NULL;" << endl;
1921 out << " }" << endl;
1922 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1923 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1924 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1925 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1926 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1927 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1928 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1929 out << " // Check if the cuts configuration file was provided" << endl;
1930 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1931 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1932 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1933 out << " }" << endl;
0df6ccf2 1934 if (fFriendChainName=="") {
1935 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1936 } else {
1937 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1938 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1939 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1940 }
c57f56b7 1941 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1942 out << " chain->ls();" << endl;
1943 out << " return chain;" << endl;
fcc9bb6f 1944 out << "}" << endl << endl;
c57f56b7 1945 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1946 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1947 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1948 msg += " AliLHCTagCuts *lhcCuts,\n";
1949 msg += " AliDetectorTagCuts *detCuts,\n";
1950 msg += " AliEventTagCuts *evCuts)";
1951 Info("WriteAnalysisMacro", msg.Data());
1952 }
0df6ccf2 1953 }
1954 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1955 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1956 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1957 out << "{" << endl;
1958 out << "// Create a chain using url's from xml file" << endl;
1959 out << " TString treename = type;" << endl;
1960 out << " treename.ToLower();" << endl;
1961 out << " treename += \"Tree\";" << endl;
e02fee64 1962 out << " printf(\"***************************************\\n\");" << endl;
1963 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1964 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1965 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1966 out << " if (!coll) {" << endl;
1967 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1968 out << " return NULL;" << endl;
1969 out << " }" << endl;
1970 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1971 if(fFriendChainName!="") {
1972 out << " TChain *chainFriend = new TChain(treename);" << endl;
1973 }
c57f56b7 1974 out << " coll->Reset();" << endl;
0df6ccf2 1975 out << " while (coll->Next()) {" << endl;
1976 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1977 if(fFriendChainName!="") {
1978 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1979 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1980 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1981 out << " chainFriend->Add(fileFriend.Data());" << endl;
1982 }
1983 out << " }" << endl;
c57f56b7 1984 out << " if (!chain->GetNtrees()) {" << endl;
1985 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1986 out << " return NULL;" << endl;
1987 out << " }" << endl;
0df6ccf2 1988 if(fFriendChainName!="") {
1989 out << " chain->AddFriend(chainFriend);" << endl;
1990 }
c57f56b7 1991 out << " return chain;" << endl;
fcc9bb6f 1992 out << "}" << endl << endl;
c57f56b7 1993 }
4e5c5506 1994 if (fPackages) {
fcc9bb6f 1995 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1996 out << "Bool_t SetupPar(const char *package) {" << endl;
1997 out << "// Compile the package and set it up." << endl;
1998 out << " TString pkgdir = package;" << endl;
1999 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 2000 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 2001 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2002 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2003 out << " // Check for BUILD.sh and execute" << endl;
2004 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2005 out << " printf(\"*******************************\\n\");" << endl;
2006 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2007 out << " printf(\"*******************************\\n\");" << endl;
2008 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 2009 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 2010 out << " gSystem->ChangeDirectory(cdir);" << endl;
2011 out << " return kFALSE;" << endl;
2012 out << " }" << endl;
2013 out << " } else {" << endl;
fcc9bb6f 2014 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 2015 out << " gSystem->ChangeDirectory(cdir);" << endl;
2016 out << " return kFALSE;" << endl;
2017 out << " }" << endl;
2018 out << " // Check for SETUP.C and execute" << endl;
2019 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2020 out << " printf(\"*******************************\\n\");" << endl;
2021 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2022 out << " printf(\"*******************************\\n\");" << endl;
2023 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2024 out << " } else {" << endl;
fcc9bb6f 2025 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 2026 out << " gSystem->ChangeDirectory(cdir);" << endl;
2027 out << " return kFALSE;" << endl;
2028 out << " }" << endl;
2029 out << " // Restore original workdir" << endl;
2030 out << " gSystem->ChangeDirectory(cdir);" << endl;
2031 out << " return kTRUE;" << endl;
2032 out << "}" << endl;
2033 }
c57f56b7 2034 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2035 }
2036 Bool_t copy = kTRUE;
2037 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2038 if (copy) {
2039 CdWork();
2040 TString workdir = gGrid->GetHomeDirectory();
2041 workdir += fGridWorkingDir;
2042 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2043 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2044 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2045 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2046 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2047 }
2048 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2049 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2050 }
2051}
2052
2053//______________________________________________________________________________
2054void AliAnalysisAlien::WriteExecutable()
2055{
2056// Generate the alien executable script.
2057 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2058 ofstream out;
2059 out.open(fExecutable.Data(), ios::out);
2060 if (out.bad()) {
5513444a 2061 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2062 return;
2063 }
2064 out << "#!/bin/bash" << endl;
c57f56b7 2065 out << "echo \"=========================================\"" << endl;
2066 out << "echo \"############## PATH : ##############\"" << endl;
2067 out << "echo $PATH" << endl;
2068 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2069 out << "echo $LD_LIBRARY_PATH" << endl;
2070 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2071 out << "echo $ROOTSYS" << endl;
2072 out << "echo \"############## which root : ##############\"" << endl;
2073 out << "which root" << endl;
2074 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2075 out << "echo $ALICE_ROOT" << endl;
2076 out << "echo \"############## which aliroot : ##############\"" << endl;
2077 out << "which aliroot" << endl;
9c5ddadc 2078 out << "echo \"############## system limits : ##############\"" << endl;
2079 out << "ulimit -a" << endl;
2080 out << "echo \"############## memory : ##############\"" << endl;
2081 out << "free -m" << endl;
c57f56b7 2082 out << "echo \"=========================================\"" << endl << endl;
74a53467 2083 // Make sure we can properly compile par files
2084 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2085 out << fExecutableCommand << " ";
631c0b05 2086 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2087 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2088 out << "echo \"############## memory after: ##############\"" << endl;
2089 out << "free -m" << endl;
2090 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2091 out << "dmesg | tail -n 10" << endl;
c57f56b7 2092 }
2093 Bool_t copy = kTRUE;
2094 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2095 if (copy) {
2096 CdWork();
2097 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2098 TString bindir = Form("%s/bin", workdir.Data());
2099 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2100 workdir += fGridWorkingDir;
2101 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2102 if (FileExists(executable)) gGrid->Rm(executable);
2103 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2104 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2105 }
2106}
2107
2108//______________________________________________________________________________
5513444a 2109void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2110{
2111// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2112// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2113// Next lines: full_path_to_dataset XXX (XXX is a string)
2114// To submit, one has to: submit jdl XXX for all lines
2115 ofstream out;
2116 out.open(filename, ios::out);
2117 if (out.bad()) {
2118 Error("WriteProductionFile", "Bad file name: %s", filename);
2119 return;
2120 }
2121 TString workdir = gGrid->GetHomeDirectory();
2122 workdir += fGridWorkingDir;
f5e8c702 2123 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2124 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2125 out << locjdl << " " << njobspermaster << endl;
5513444a 2126 Int_t nmasterjobs = fInputFiles->GetEntries();
2127 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2128 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2129 }
2130 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2131 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2132}
2133
2134//______________________________________________________________________________
c57f56b7 2135void AliAnalysisAlien::WriteValidationScript()
2136{
2137// Generate the alien validation script.
2138 // Generate the validation script
2139 TObjString *os;
0d5d317c 2140 TString validationScript = fExecutable;
2141 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2142 if (!Connect()) {
2143 Error("WriteValidationScript", "Alien connection required");
2144 return;
2145 }
2146 TString out_stream = "";
2147 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2148 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2149 ofstream out;
0d5d317c 2150 out.open(validationScript, ios::out);
c57f56b7 2151 out << "#!/bin/bash" << endl;
2152 out << "##################################################" << endl;
2153 out << "validateout=`dirname $0`" << endl;
2154 out << "validatetime=`date`" << endl;
2155 out << "validated=\"0\";" << endl;
2156 out << "error=0" << endl;
2157 out << "if [ -z $validateout ]" << endl;
2158 out << "then" << endl;
2159 out << " validateout=\".\"" << endl;
2160 out << "fi" << endl << endl;
2161 out << "cd $validateout;" << endl;
2162 out << "validateworkdir=`pwd`;" << endl << endl;
2163 out << "echo \"*******************************************************\"" << out_stream << endl;
2164 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2165 out << "" << endl;
2166 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2167 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2168 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2169 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2170 out << "ls -la ./" << out_stream << endl;
2171 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2172 out << "##################################################" << endl;
ebec370a 2173
2174 out << "" << endl;
2175 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2176 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2177 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2178 out << "" << endl;
2179
2180 out << "if [ ! -f stderr ] ; then" << endl;
2181 out << " error=1" << endl;
2182 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2183 out << " echo \"Error = $error\" " << out_stream << endl;
2184 out << "fi" << endl;
2185
2186 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2187 out << " error=1" << endl;
2188 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2189 out << " echo \"$parArch\" " << out_stream << endl;
2190 out << " echo \"Error = $error\" " << out_stream << endl;
2191 out << "fi" << endl;
2192
2193 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2194 out << " error=1" << endl;
2195 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2196 out << " echo \"$segViol\" " << out_stream << endl;
2197 out << " echo \"Error = $error\" " << out_stream << endl;
2198 out << "fi" << endl;
2199
2200 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2201 out << " error=1" << endl;
2202 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2203 out << " echo \"$segFault\" " << out_stream << endl;
2204 out << " echo \"Error = $error\" " << out_stream << endl;
2205 out << "fi" << endl;
2206
2207 // Part dedicated to the specific analyses running into the train
2208
c57f56b7 2209 TObjArray *arr = fOutputFiles.Tokenize(" ");
2210 TIter next1(arr);
2211 TString output_file;
2212 while ((os=(TObjString*)next1())) {
2213 output_file = os->GetString();
2214 Int_t index = output_file.Index("@");
2215 if (index > 0) output_file.Remove(index);
2216 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2217 out << " error=1" << endl;
2218 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2219 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2220 out << "fi" << endl;
2221 }
2222 delete arr;
923e2ca5 2223 out << "if ! [ -f outputs_valid ] ; then" << endl;
2224 out << " error=1" << endl;
2225 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2226 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2227 out << "fi" << endl;
2228
c57f56b7 2229 out << "if [ $error = 0 ] ; then" << endl;
2230 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2231 out << "fi" << endl;
2232
2233 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2234 out << "echo \"*******************************************************\"" << out_stream << endl;
2235 out << "cd -" << endl;
2236 out << "exit $error" << endl;
2237 }
2238 Bool_t copy = kTRUE;
2239 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2240 if (copy) {
2241 CdWork();
2242 TString workdir = gGrid->GetHomeDirectory();
2243 workdir += fGridWorkingDir;
0d5d317c 2244 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2245 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2246 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2247 }
2248}