]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
increase classdef
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
cd11251e 57 fOutputToRunNo(0),
c57f56b7 58 fRunNumbers(),
59 fExecutable(),
0a1c1f7f 60 fExecutableCommand(),
c57f56b7 61 fArguments(),
631c0b05 62 fExecutableArgs(),
c57f56b7 63 fAnalysisMacro(),
64 fAnalysisSource(),
d5c6455a 65 fAdditionalRootLibs(),
c57f56b7 66 fAdditionalLibs(),
67 fSplitMode(),
68 fAPIVersion(),
69 fROOTVersion(),
70 fAliROOTVersion(),
648174cf 71 fExternalPackages(),
c57f56b7 72 fUser(),
73 fGridWorkingDir(),
74 fGridDataDir(),
75 fDataPattern(),
76 fGridOutputDir(),
77 fOutputArchive(),
78 fOutputFiles(),
79 fInputFormat(),
e7c71df0 80 fDatasetName(),
c57f56b7 81 fJDLName(),
bb885a9e 82 fMergeExcludes(),
f965131e 83 fIncludePath(),
bb885a9e 84 fCloseSE(),
0df6ccf2 85 fFriendChainName(),
c6cb3634 86 fJobTag(),
648174cf 87 fOutputSingle(),
5fce53f4 88 fRunPrefix(),
4e5c5506 89 fInputFiles(0),
90 fPackages(0)
c57f56b7 91{
92// Dummy ctor.
93 SetDefaults();
94}
95
96//______________________________________________________________________________
97AliAnalysisAlien::AliAnalysisAlien(const char *name)
98 :AliAnalysisGrid(name),
99 fGridJDL(NULL),
100 fPrice(0),
101 fTTL(0),
102 fSplitMaxInputFileNumber(0),
103 fMaxInitFailed(0),
104 fMasterResubmitThreshold(0),
bb885a9e 105 fNtestFiles(0),
319593fb 106 fNrunsPerMaster(0),
16a4353c 107 fMaxMergeFiles(0),
d2a409b2 108 fNsubmitted(0),
a3e84053 109 fProductionMode(0),
cd11251e 110 fOutputToRunNo(0),
c57f56b7 111 fRunNumbers(),
112 fExecutable(),
0a1c1f7f 113 fExecutableCommand(),
c57f56b7 114 fArguments(),
631c0b05 115 fExecutableArgs(),
c57f56b7 116 fAnalysisMacro(),
117 fAnalysisSource(),
d5c6455a 118 fAdditionalRootLibs(),
c57f56b7 119 fAdditionalLibs(),
120 fSplitMode(),
121 fAPIVersion(),
122 fROOTVersion(),
123 fAliROOTVersion(),
648174cf 124 fExternalPackages(),
c57f56b7 125 fUser(),
126 fGridWorkingDir(),
127 fGridDataDir(),
128 fDataPattern(),
129 fGridOutputDir(),
130 fOutputArchive(),
131 fOutputFiles(),
132 fInputFormat(),
e7c71df0 133 fDatasetName(),
c57f56b7 134 fJDLName(),
bb885a9e 135 fMergeExcludes(),
f965131e 136 fIncludePath(),
bb885a9e 137 fCloseSE(),
0df6ccf2 138 fFriendChainName(),
c6cb3634 139 fJobTag(),
648174cf 140 fOutputSingle(),
5fce53f4 141 fRunPrefix(),
4e5c5506 142 fInputFiles(0),
143 fPackages(0)
c57f56b7 144{
145// Default ctor.
146 SetDefaults();
147}
148
149//______________________________________________________________________________
150AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
151 :AliAnalysisGrid(other),
152 fGridJDL(NULL),
153 fPrice(other.fPrice),
154 fTTL(other.fTTL),
155 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
156 fMaxInitFailed(other.fMaxInitFailed),
157 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 158 fNtestFiles(other.fNtestFiles),
319593fb 159 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 160 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 161 fNsubmitted(other.fNsubmitted),
a3e84053 162 fProductionMode(other.fProductionMode),
cd11251e 163 fOutputToRunNo(other.fOutputToRunNo),
c57f56b7 164 fRunNumbers(other.fRunNumbers),
165 fExecutable(other.fExecutable),
0a1c1f7f 166 fExecutableCommand(other.fExecutableCommand),
c57f56b7 167 fArguments(other.fArguments),
631c0b05 168 fExecutableArgs(other.fExecutableArgs),
c57f56b7 169 fAnalysisMacro(other.fAnalysisMacro),
170 fAnalysisSource(other.fAnalysisSource),
d5c6455a 171 fAdditionalRootLibs(other.fAdditionalRootLibs),
c57f56b7 172 fAdditionalLibs(other.fAdditionalLibs),
173 fSplitMode(other.fSplitMode),
174 fAPIVersion(other.fAPIVersion),
175 fROOTVersion(other.fROOTVersion),
176 fAliROOTVersion(other.fAliROOTVersion),
648174cf 177 fExternalPackages(other.fExternalPackages),
c57f56b7 178 fUser(other.fUser),
179 fGridWorkingDir(other.fGridWorkingDir),
180 fGridDataDir(other.fGridDataDir),
181 fDataPattern(other.fDataPattern),
182 fGridOutputDir(other.fGridOutputDir),
183 fOutputArchive(other.fOutputArchive),
184 fOutputFiles(other.fOutputFiles),
185 fInputFormat(other.fInputFormat),
e7c71df0 186 fDatasetName(other.fDatasetName),
c57f56b7 187 fJDLName(other.fJDLName),
bb885a9e 188 fMergeExcludes(other.fMergeExcludes),
f965131e 189 fIncludePath(other.fIncludePath),
bb885a9e 190 fCloseSE(other.fCloseSE),
0df6ccf2 191 fFriendChainName(other.fFriendChainName),
c6cb3634 192 fJobTag(other.fJobTag),
648174cf 193 fOutputSingle(other.fOutputSingle),
5fce53f4 194 fRunPrefix(other.fRunPrefix),
4e5c5506 195 fInputFiles(0),
196 fPackages(0)
c57f56b7 197{
198// Copy ctor.
199 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 200 fRunRange[0] = other.fRunRange[0];
201 fRunRange[1] = other.fRunRange[1];
c57f56b7 202 if (other.fInputFiles) {
203 fInputFiles = new TObjArray();
204 TIter next(other.fInputFiles);
205 TObject *obj;
206 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
207 fInputFiles->SetOwner();
208 }
4e5c5506 209 if (other.fPackages) {
210 fPackages = new TObjArray();
211 TIter next(other.fPackages);
212 TObject *obj;
213 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
214 fPackages->SetOwner();
215 }
c57f56b7 216}
217
218//______________________________________________________________________________
219AliAnalysisAlien::~AliAnalysisAlien()
220{
221// Destructor.
222 if (fGridJDL) delete fGridJDL;
223 if (fInputFiles) delete fInputFiles;
4e5c5506 224 if (fPackages) delete fPackages;
c57f56b7 225}
226
227//______________________________________________________________________________
228AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
229{
230// Assignment.
231 if (this != &other) {
232 AliAnalysisGrid::operator=(other);
233 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
234 fPrice = other.fPrice;
235 fTTL = other.fTTL;
236 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
237 fMaxInitFailed = other.fMaxInitFailed;
238 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 239 fNtestFiles = other.fNtestFiles;
a3e84053 240 fNrunsPerMaster = other.fNrunsPerMaster;
241 fMaxMergeFiles = other.fMaxMergeFiles;
242 fNsubmitted = other.fNsubmitted;
243 fProductionMode = other.fProductionMode;
cd11251e 244 fOutputToRunNo = other.fOutputToRunNo;
c57f56b7 245 fRunNumbers = other.fRunNumbers;
246 fExecutable = other.fExecutable;
0a1c1f7f 247 fExecutableCommand = other.fExecutableCommand;
c57f56b7 248 fArguments = other.fArguments;
631c0b05 249 fExecutableArgs = other.fExecutableArgs;
c57f56b7 250 fAnalysisMacro = other.fAnalysisMacro;
251 fAnalysisSource = other.fAnalysisSource;
d5c6455a 252 fAdditionalRootLibs = other.fAdditionalRootLibs;
c57f56b7 253 fAdditionalLibs = other.fAdditionalLibs;
254 fSplitMode = other.fSplitMode;
255 fAPIVersion = other.fAPIVersion;
256 fROOTVersion = other.fROOTVersion;
257 fAliROOTVersion = other.fAliROOTVersion;
648174cf 258 fExternalPackages = other.fExternalPackages;
c57f56b7 259 fUser = other.fUser;
260 fGridWorkingDir = other.fGridWorkingDir;
261 fGridDataDir = other.fGridDataDir;
262 fDataPattern = other.fDataPattern;
263 fGridOutputDir = other.fGridOutputDir;
264 fOutputArchive = other.fOutputArchive;
265 fOutputFiles = other.fOutputFiles;
266 fInputFormat = other.fInputFormat;
e7c71df0 267 fDatasetName = other.fDatasetName;
c57f56b7 268 fJDLName = other.fJDLName;
bb885a9e 269 fMergeExcludes = other.fMergeExcludes;
f965131e 270 fIncludePath = other.fIncludePath;
bb885a9e 271 fCloseSE = other.fCloseSE;
0df6ccf2 272 fFriendChainName = other.fFriendChainName;
c6cb3634 273 fJobTag = other.fJobTag;
648174cf 274 fOutputSingle = other.fOutputSingle;
5fce53f4 275 fRunPrefix = other.fRunPrefix;
c57f56b7 276 if (other.fInputFiles) {
277 fInputFiles = new TObjArray();
278 TIter next(other.fInputFiles);
279 TObject *obj;
280 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
281 fInputFiles->SetOwner();
282 }
4e5c5506 283 if (other.fPackages) {
284 fPackages = new TObjArray();
285 TIter next(other.fPackages);
286 TObject *obj;
287 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
288 fPackages->SetOwner();
289 }
c57f56b7 290 }
291 return *this;
292}
293
f965131e 294//______________________________________________________________________________
295void AliAnalysisAlien::AddIncludePath(const char *path)
296{
297// Add include path in the remote analysis macro.
298 TString p(path);
299 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
300 else fIncludePath += Form("-I%s ", path);
301}
302
c57f56b7 303//______________________________________________________________________________
304void AliAnalysisAlien::AddRunNumber(Int_t run)
305{
306// Add a run number to the list of runs to be processed.
307 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 308 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 309}
310
ee75cfc3 311//______________________________________________________________________________
312void AliAnalysisAlien::AddRunNumber(const char* run)
313{
314// Add a run number to the list of runs to be processed.
315 if (fRunNumbers.Length()) fRunNumbers += " ";
316 fRunNumbers += run;
317}
318
c57f56b7 319//______________________________________________________________________________
320void AliAnalysisAlien::AddDataFile(const char *lfn)
321{
322// Adds a data file to the input to be analysed. The file should be a valid LFN
323// or point to an existing file in the alien workdir.
324 if (!fInputFiles) fInputFiles = new TObjArray();
325 fInputFiles->Add(new TObjString(lfn));
326}
648174cf 327
328//______________________________________________________________________________
329void AliAnalysisAlien::AddExternalPackage(const char *package)
330{
331// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
332 if (fExternalPackages) fExternalPackages += " ";
333 fExternalPackages += package;
334}
335
c57f56b7 336//______________________________________________________________________________
337Bool_t AliAnalysisAlien::Connect()
338{
339// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
340 if (gGrid && gGrid->IsConnected()) return kTRUE;
341 if (!gSystem->Getenv("alien_API_USER")) {
342 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
343 gSystem->Getenv("UID"));
344 return kFALSE;
345 }
346 if (!gGrid) {
347 Info("Connect", "Trying to connect to AliEn ...");
348 TGrid::Connect("alien://");
349 }
350 if (!gGrid || !gGrid->IsConnected()) {
351 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
352 return kFALSE;
353 }
354 fUser = gGrid->GetUser();
355 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
356 return kTRUE;
357}
358
359//______________________________________________________________________________
360void AliAnalysisAlien::CdWork()
361{
362// Check validity of alien workspace. Create directory if possible.
363 if (!Connect()) {
364 Error("CdWork", "Alien connection required");
365 return;
366 }
367 TString homedir = gGrid->GetHomeDirectory();
368 TString workdir = homedir + fGridWorkingDir;
923e2ca5 369 if (DirectoryExists(workdir)) {
370 gGrid->Cd(workdir);
371 return;
372 }
373 // Work directory not existing - create it
374 gGrid->Cd(homedir);
375 if (gGrid->Mkdir(workdir)) {
376 gGrid->Cd(fGridWorkingDir);
377 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
378 } else {
379 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
380 workdir.Data(), homedir.Data());
381 fGridWorkingDir = "";
382 }
c57f56b7 383}
384
385//______________________________________________________________________________
386Bool_t AliAnalysisAlien::CheckInputData()
387{
388// Check validity of input data. If necessary, create xml files.
a8739e8a 389 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 390 if (!fGridDataDir.Length()) {
391 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
392 return kFALSE;
393 }
394 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
395 return kTRUE;
c57f56b7 396 }
397 // Process declared files
398 Bool_t is_collection = kFALSE;
399 Bool_t is_xml = kFALSE;
400 Bool_t use_tags = kFALSE;
401 Bool_t checked = kFALSE;
402 CdWork();
403 TString file;
404 TString workdir = gGrid->GetHomeDirectory();
405 workdir += fGridWorkingDir;
406 if (fInputFiles) {
407 TObjString *objstr;
408 TIter next(fInputFiles);
409 while ((objstr=(TObjString*)next())) {
410 file = workdir;
411 file += "/";
412 file += objstr->GetString();
413 // Store full lfn path
414 if (FileExists(file)) objstr->SetString(file);
415 else {
416 file = objstr->GetName();
417 if (!FileExists(objstr->GetName())) {
418 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
419 objstr->GetName(), workdir.Data());
420 return kFALSE;
421 }
422 }
423 Bool_t iscoll, isxml, usetags;
424 CheckDataType(file, iscoll, isxml, usetags);
425 if (!checked) {
426 checked = kTRUE;
427 is_collection = iscoll;
428 is_xml = isxml;
429 use_tags = usetags;
430 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
431 } else {
432 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
433 Error("CheckInputData", "Some conflict was found in the types of inputs");
434 return kFALSE;
435 }
436 }
437 }
438 }
439 // Process requested run numbers
a8739e8a 440 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 441 // Check validity of alien data directory
442 if (!fGridDataDir.Length()) {
443 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
444 return kFALSE;
445 }
923e2ca5 446 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 447 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
448 return kFALSE;
449 }
450 if (is_collection) {
451 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
452 return kFALSE;
453 }
454
455 if (checked && !is_xml) {
456 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
457 return kFALSE;
458 }
459 // Check validity of run number(s)
460 TObjArray *arr;
461 TObjString *os;
319593fb 462 Int_t nruns = 0;
904f9f5f 463 TString schunk, schunk2;
c57f56b7 464 TString path;
465 if (!checked) {
466 checked = kTRUE;
467 use_tags = fDataPattern.Contains("tag");
468 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
469 }
470 if (use_tags != fDataPattern.Contains("tag")) {
471 Error("CheckInputData", "Cannot mix input files using/not using tags");
472 return kFALSE;
473 }
474 if (fRunNumbers.Length()) {
a8739e8a 475 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 476 arr = fRunNumbers.Tokenize(" ");
477 TIter next(arr);
478 while ((os=(TObjString*)next())) {
479 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 480 if (!DirectoryExists(path)) {
481 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 482 continue;
c57f56b7 483 }
484 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
485 TString msg = "\n##### file: ";
486 msg += path;
487 msg += " type: xml_collection;";
488 if (use_tags) msg += " using_tags: Yes";
489 else msg += " using_tags: No";
490 Info("CheckDataType", msg.Data());
319593fb 491 if (fNrunsPerMaster<2) {
d2a409b2 492 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 493 } else {
494 nruns++;
495 if (((nruns-1)%fNrunsPerMaster) == 0) {
496 schunk = os->GetString();
497 }
498 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
499 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 500 AddDataFile(schunk);
319593fb 501 }
c57f56b7 502 }
503 delete arr;
a8739e8a 504 } else {
505 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
506 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 507 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 508 if (!DirectoryExists(path)) {
509// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 510 continue;
511 }
5fce53f4 512 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 513 TString msg = "\n##### file: ";
514 msg += path;
515 msg += " type: xml_collection;";
516 if (use_tags) msg += " using_tags: Yes";
517 else msg += " using_tags: No";
518 Info("CheckDataType", msg.Data());
319593fb 519 if (fNrunsPerMaster<2) {
5fce53f4 520 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 521 } else {
522 nruns++;
523 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 524 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 525 }
904f9f5f 526 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 527 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 528 schunk += schunk2;
d2a409b2 529 AddDataFile(schunk);
319593fb 530 }
a8739e8a 531 }
904f9f5f 532 if (!fInputFiles) {
533 schunk += schunk2;
534 AddDataFile(schunk);
535 }
c57f56b7 536 }
537 return kTRUE;
538}
539
540//______________________________________________________________________________
541Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
542{
543// Create dataset for the grid data directory + run number.
544 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
545 if (!Connect()) {
546 Error("CreateDataset", "Cannot create dataset with no grid connection");
547 return kFALSE;
548 }
549
550 // Cd workspace
551 CdWork();
552 TString workdir = gGrid->GetHomeDirectory();
553 workdir += fGridWorkingDir;
554
555 // Compose the 'find' command arguments
556 TString command;
557 TString options = "-x collection ";
bb885a9e 558 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 559 TString conditions = "";
560
561 TString file;
562 TString path;
319593fb 563 Int_t nruns = 0;
904f9f5f 564 TString schunk, schunk2;
ab254fd1 565 TGridCollection *cbase=0, *cadd=0;
d2a409b2 566 if (!fRunNumbers.Length() && !fRunRange[0]) {
567 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
568 // Make a single data collection from data directory.
569 path = fGridDataDir;
923e2ca5 570 if (!DirectoryExists(path)) {
d2a409b2 571 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
572 return kFALSE;
573 }
923e2ca5 574// CdWork();
d2a409b2 575 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
576 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 577 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 578 command = "find ";
579 command += options;
580 command += path;
581 command += " ";
582 command += pattern;
583 command += conditions;
84fcd93f 584 printf("command: %s\n", command.Data());
d2a409b2 585 TGridResult *res = gGrid->Command(command);
586 if (res) delete res;
587 // Write standard output to file
588 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
589 }
590 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
591 // Copy xml file to alien space
592 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
593 if (!FileExists(file)) {
594 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
595 return kFALSE;
596 }
597 // Update list of files to be processed.
598 }
599 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
600 return kTRUE;
601 }
c57f56b7 602 // Several runs
a8739e8a 603 if (fRunNumbers.Length()) {
604 TObjArray *arr = fRunNumbers.Tokenize(" ");
605 TObjString *os;
606 TIter next(arr);
607 while ((os=(TObjString*)next())) {
608 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 609 if (!DirectoryExists(path)) continue;
610// CdWork();
a8739e8a 611 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
612 else file = Form("%s.xml", os->GetString().Data());
319593fb 613 // If local collection file does not exist, create it via 'find' command.
614 if (gSystem->AccessPathName(file)) {
615 command = "find ";
616 command += options;
617 command += path;
618 command += pattern;
619 command += conditions;
620 TGridResult *res = gGrid->Command(command);
621 if (res) delete res;
622 // Write standard output to file
623 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
624 }
a8739e8a 625 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 626 // Check if there is one run per master job.
627 if (fNrunsPerMaster<2) {
628 if (FileExists(file)) {
629 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
630 continue;
631 }
632 // Copy xml file to alien space
633 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
634 if (!FileExists(file)) {
635 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
636 delete arr;
637 return kFALSE;
638 }
639 } else {
640 nruns++;
641 if (((nruns-1)%fNrunsPerMaster) == 0) {
642 schunk = os->GetString();
643 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
644 } else {
645 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 646 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 647 cbase->Add(cadd);
648 delete cadd;
649 }
650 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
651 continue;
652 }
653 schunk += Form("_%s.xml", os->GetString().Data());
654 if (FileExists(schunk)) {
655 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
656 continue;
657 }
84fcd93f 658 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 659 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 660 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 661 if (!FileExists(schunk)) {
662 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
663 delete arr;
664 return kFALSE;
665 }
666 }
a8739e8a 667 }
668 delete arr;
669 } else {
670 // Process a full run range.
671 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 672 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 673 if (!DirectoryExists(path)) continue;
674// CdWork();
a8739e8a 675 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 676 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
319593fb 677 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 678 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
679// gGrid->Rm(file);
680 continue;
681 }
319593fb 682 // If local collection file does not exist, create it via 'find' command.
683 if (gSystem->AccessPathName(file)) {
684 command = "find ";
685 command += options;
686 command += path;
687 command += pattern;
688 command += conditions;
689 TGridResult *res = gGrid->Command(command);
690 if (res) delete res;
691 // Write standard output to file
692 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
693 }
a8739e8a 694 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 695 // Check if there is one run per master job.
696 if (fNrunsPerMaster<2) {
697 if (FileExists(file)) {
698 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
699 continue;
700 }
701 // Copy xml file to alien space
702 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
703 if (!FileExists(file)) {
704 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
705 return kFALSE;
706 }
707 } else {
708 nruns++;
95e5b448 709 // Check if the collection for the chunk exist locally.
710 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
711 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 712 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 713 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 714 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 715 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
716 } else {
717 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
718 cbase->Add(cadd);
719 delete cadd;
720 }
904f9f5f 721 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
722 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 723 continue;
724 }
904f9f5f 725 schunk = schunk2;
319593fb 726 if (FileExists(schunk)) {
727 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
728 continue;
729 }
84fcd93f 730 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 731 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 732 if (FileExists(schunk)) {
733 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
734 continue;
735 }
319593fb 736 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
737 if (!FileExists(schunk)) {
738 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
739 return kFALSE;
740 }
741 }
c57f56b7 742 }
a8739e8a 743 }
c57f56b7 744 return kTRUE;
745}
746
747//______________________________________________________________________________
748Bool_t AliAnalysisAlien::CreateJDL()
749{
750// Generate a JDL file according to current settings. The name of the file is
751// specified by fJDLName.
752 Bool_t error = kFALSE;
753 TObjArray *arr = 0;
754 Bool_t copy = kTRUE;
755 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
756 Bool_t generate = kTRUE;
757 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
758 if (!Connect()) {
759 Error("CreateJDL", "Alien connection required");
760 return kFALSE;
761 }
762 // Check validity of alien workspace
763 CdWork();
764 TString workdir = gGrid->GetHomeDirectory();
765 workdir += fGridWorkingDir;
766 if (generate) {
767 TObjString *os;
768 if (!fInputFiles) {
769 Error("CreateJDL()", "Define some input files for your analysis.");
770 error = kTRUE;
771 }
772 // Compose list of input files
773 // Check if output files were defined
774 if (!fOutputFiles.Length()) {
775 Error("CreateJDL", "You must define at least one output file");
776 error = kTRUE;
777 }
778 // Check if an output directory was defined and valid
779 if (!fGridOutputDir.Length()) {
780 Error("CreateJDL", "You must define AliEn output directory");
781 error = kTRUE;
782 } else {
d2a409b2 783 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 784 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 785 if (gGrid->Mkdir(fGridOutputDir)) {
786 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
787 } else {
788 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 789 // error = kTRUE;
c57f56b7 790 }
791 }
792 gGrid->Cd(workdir);
793 }
794 // Exit if any error up to now
795 if (error) return kFALSE;
796 // Set JDL fields
797 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 798 fGridJDL->SetExecutable(fExecutable);
b5b9dee8 799 if (!fArguments.IsNull())
800 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
c57f56b7 801// fGridJDL->SetTTL((UInt_t)fTTL);
802 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
803 if (fMaxInitFailed > 0)
804 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
805 if (fSplitMaxInputFileNumber > 0)
806 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
807 if (fSplitMode.Length())
808 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
809// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
810 if (fAliROOTVersion.Length())
811 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
812 if (fROOTVersion.Length())
813 fGridJDL->AddToPackages("ROOT", fROOTVersion);
814 if (fAPIVersion.Length())
815 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
648174cf 816 if (!fExternalPackages.IsNull()) {
817 arr = fExternalPackages.Tokenize(" ");
818 TIter next(arr);
819 while ((os=(TObjString*)next())) {
820 TString pkgname = os->GetString();
821 Int_t index = pkgname.Index("::");
822 TString pkgversion = pkgname(index+2, pkgname.Length());
823 pkgname.Remove(index);
824 fGridJDL->AddToPackages(pkgname, pkgversion);
825 }
826 delete arr;
827 }
c57f56b7 828 fGridJDL->SetInputDataListFormat(fInputFormat);
829 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 830 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 831 TString analysisFile = fExecutable;
832 analysisFile.ReplaceAll(".sh", ".root");
833 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 834 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
835 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
836 if (fAdditionalLibs.Length()) {
837 arr = fAdditionalLibs.Tokenize(" ");
838 TIter next(arr);
839 while ((os=(TObjString*)next())) {
840 if (os->GetString().Contains(".so")) continue;
841 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
842 }
843 delete arr;
844 }
4e5c5506 845 if (fPackages) {
846 TIter next(fPackages);
847 TObject *obj;
848 while ((obj=next()))
849 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
850 }
c57f56b7 851 if (fOutputArchive.Length()) {
852 arr = fOutputArchive.Tokenize(" ");
853 TIter next(arr);
854 while ((os=(TObjString*)next()))
43da816a 855 if (!os->GetString().Contains("@") && fCloseSE.Length())
856 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
857 else
858 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 859 delete arr;
860 }
c57f56b7 861 arr = fOutputFiles.Tokenize(" ");
862 TIter next(arr);
43da816a 863 while ((os=(TObjString*)next())) {
e1eaf596 864 // Ignore ouputs in jdl that are also in outputarchive
865 TString sout = os->GetString();
866 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
867 if (fOutputArchive.Contains(sout)) continue;
43da816a 868 if (!os->GetString().Contains("@") && fCloseSE.Length())
869 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
870 else
871 fGridJDL->AddToOutputSandbox(os->GetString());
872 }
c57f56b7 873 delete arr;
874// fGridJDL->SetPrice((UInt_t)fPrice);
875 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 876 TString validationScript = fExecutable;
877 validationScript.ReplaceAll(".sh", "_validation.sh");
878 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 879 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 880 // Write a jdl with 2 input parameters: collection name and output dir name.
881 WriteJDL(copy);
c57f56b7 882 }
883 // Copy jdl to grid workspace
a8739e8a 884 if (copy) {
b5e4aaa7 885 // Check if an output directory was defined and valid
886 if (!fGridOutputDir.Length()) {
887 Error("CreateJDL", "You must define AliEn output directory");
888 return kFALSE;
889 } else {
890 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
891 if (!DirectoryExists(fGridOutputDir)) {
892 if (gGrid->Mkdir(fGridOutputDir)) {
893 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
894 } else {
895 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
896 return kFALSE;
897 }
898 }
899 gGrid->Cd(workdir);
900 }
648174cf 901 if (TestBit(AliAnalysisGrid::kSubmit)) {
902 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
903 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
904 if (fProductionMode)
905 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
906 if (FileExists(locjdl)) gGrid->Rm(locjdl);
907 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
908 }
c57f56b7 909 if (fAdditionalLibs.Length()) {
e7c71df0 910 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 911 TObjString *os;
912 TIter next(arr);
913 while ((os=(TObjString*)next())) {
c57f56b7 914 if (os->GetString().Contains(".so")) continue;
4e5c5506 915 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 916 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
917 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
918 }
919 delete arr;
920 }
4e5c5506 921 if (fPackages) {
922 TIter next(fPackages);
923 TObject *obj;
924 while ((obj=next())) {
925 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
926 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
927 }
928 }
c57f56b7 929 }
930 return kTRUE;
931}
932
a8739e8a 933//______________________________________________________________________________
d2a409b2 934Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 935{
936// Writes one or more JDL's corresponding to findex. If findex is negative,
937// all run numbers are considered in one go (jdl). For non-negative indices
938// they correspond to the indices in the array fInputFiles.
939 if (!fInputFiles) return kFALSE;
940 TObjString *os;
a8739e8a 941 TString workdir = gGrid->GetHomeDirectory();
942 workdir += fGridWorkingDir;
d2a409b2 943
944 if (!fRunNumbers.Length() && !fRunRange[0]) {
945 // One jdl with no parameters in case input data is specified by name.
a8739e8a 946 TIter next(fInputFiles);
947 while ((os=(TObjString*)next()))
948 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
648174cf 949 if (!fOutputSingle.IsNull())
79a1a3f2 950 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()));
648174cf 951 else
952 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 953 } else {
d2a409b2 954 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
955 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
cd11251e 956 if (!fOutputSingle.IsNull()) {
957 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
958 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()));
959 } else {
960 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
961 }
a8739e8a 962 }
963
964
965 // Generate the JDL as a string
966 TString sjdl = fGridJDL->Generate();
967 Int_t index;
968 index = sjdl.Index("Executable");
969 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
970 index = sjdl.Index("Split ");
648174cf 971 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
a8739e8a 972 index = sjdl.Index("SplitMaxInputFileNumber");
973 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
974 index = sjdl.Index("InputDataCollection");
975 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 976 index = sjdl.Index("InputFile");
977 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
978 index = sjdl.Index("InputDataList ");
979 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
980 index = sjdl.Index("InputDataListFormat");
981 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
982 index = sjdl.Index("Price");
983 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
984 index = sjdl.Index("Requirements");
985 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
986 index = sjdl.Index("Packages");
987 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
988 index = sjdl.Index("User =");
989 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
990 index = sjdl.Index("TTL");
991 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
992 index = sjdl.Index("OutputFile");
993 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
994 index = sjdl.Index("OutputDir");
995 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
996 index = sjdl.Index("OutputArchive");
997 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
998 index = sjdl.Index("MaxInitFailed");
999 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
1000 index = sjdl.Index("MasterResubmitThreshold");
1001 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
1002 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
1003 index = sjdl.Index("Validationcommand");
1004 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
1005 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1006 sjdl.ReplaceAll("(member", "\n (member");
1007 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1008 sjdl.ReplaceAll("{", "{\n ");
1009 sjdl.ReplaceAll("};", "\n};");
1010 sjdl.ReplaceAll("{\n \n", "{\n");
1011 sjdl.ReplaceAll("\n\n", "\n");
1012 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
1013 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 1014 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 1015 index = sjdl.Index("JDLVariables");
1016 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
1017 // Write jdl to file
a8739e8a 1018 ofstream out;
d2a409b2 1019 out.open(fJDLName.Data(), ios::out);
a8739e8a 1020 if (out.bad()) {
d2a409b2 1021 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 1022 return kFALSE;
1023 }
1024 out << sjdl << endl;
1025
1026 // Copy jdl to grid workspace
1027 if (!copy) {
1028 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1029 } else {
d2a409b2 1030 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1031 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 1032 if (fProductionMode)
b5fe9cba 1033 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 1034 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 1035 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 1036 }
1037 return kTRUE;
1038}
1039
c57f56b7 1040//______________________________________________________________________________
5513444a 1041Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1042{
1043// Returns true if file exists.
5513444a 1044 if (!gGrid) return kFALSE;
c57f56b7 1045 TGridResult *res = gGrid->Ls(lfn);
1046 if (!res) return kFALSE;
1047 TMap *map = dynamic_cast<TMap*>(res->At(0));
1048 if (!map) {
1049 delete res;
1050 return kFALSE;
1051 }
1052 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1053 if (!objs || !objs->GetString().Length()) {
1054 delete res;
1055 return kFALSE;
1056 }
1057 delete res;
1058 return kTRUE;
1059}
1060
923e2ca5 1061//______________________________________________________________________________
1062Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1063{
1064// Returns true if directory exists. Can be also a path.
1065 if (!gGrid) return kFALSE;
1066 // Check if dirname is a path
1067 TString dirstripped = dirname;
1068 dirstripped = dirstripped.Strip();
1069 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1070 TString dir = gSystem->BaseName(dirstripped);
1071 dir += "/";
1072 TString path = gSystem->DirName(dirstripped);
1073 TGridResult *res = gGrid->Ls(path, "-F");
1074 if (!res) return kFALSE;
1075 TIter next(res);
1076 TMap *map;
1077 TObject *obj;
1078 while ((map=dynamic_cast<TMap*>(next()))) {
1079 obj = map->GetValue("name");
1080 if (!obj) break;
1081 if (dir == obj->GetName()) {
1082 delete res;
1083 return kTRUE;
1084 }
1085 }
1086 delete res;
1087 return kFALSE;
1088}
1089
c57f56b7 1090//______________________________________________________________________________
1091void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1092{
1093// Check input data type.
1094 is_collection = kFALSE;
1095 is_xml = kFALSE;
1096 use_tags = kFALSE;
1097 if (!gGrid) {
1098 Error("CheckDataType", "No connection to grid");
1099 return;
1100 }
1101 is_collection = IsCollection(lfn);
1102 TString msg = "\n##### file: ";
1103 msg += lfn;
1104 if (is_collection) {
1105 msg += " type: raw_collection;";
1106 // special treatment for collections
1107 is_xml = kFALSE;
1108 // check for tag files in the collection
1109 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1110 if (!res) {
1111 msg += " using_tags: No (unknown)";
1112 Info("CheckDataType", msg.Data());
1113 return;
1114 }
1115 const char* typeStr = res->GetKey(0, "origLFN");
1116 if (!typeStr || !strlen(typeStr)) {
1117 msg += " using_tags: No (unknown)";
1118 Info("CheckDataType", msg.Data());
1119 return;
1120 }
1121 TString file = typeStr;
1122 use_tags = file.Contains(".tag");
1123 if (use_tags) msg += " using_tags: Yes";
1124 else msg += " using_tags: No";
1125 Info("CheckDataType", msg.Data());
1126 return;
1127 }
1128 TString slfn(lfn);
1129 slfn.ToLower();
1130 is_xml = slfn.Contains(".xml");
1131 if (is_xml) {
1132 // Open xml collection and check if there are tag files inside
1133 msg += " type: xml_collection;";
1134 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1135 if (!coll) {
1136 msg += " using_tags: No (unknown)";
1137 Info("CheckDataType", msg.Data());
1138 return;
1139 }
1140 TMap *map = coll->Next();
1141 if (!map) {
1142 msg += " using_tags: No (unknown)";
1143 Info("CheckDataType", msg.Data());
1144 return;
1145 }
1146 map = (TMap*)map->GetValue("");
1147 TString file;
1148 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1149 use_tags = file.Contains(".tag");
1150 delete coll;
1151 if (use_tags) msg += " using_tags: Yes";
1152 else msg += " using_tags: No";
1153 Info("CheckDataType", msg.Data());
1154 return;
1155 }
1156 use_tags = slfn.Contains(".tag");
1157 if (slfn.Contains(".root")) msg += " type: root file;";
1158 else msg += " type: unhnown file;";
1159 if (use_tags) msg += " using_tags: Yes";
1160 else msg += " using_tags: No";
1161 Info("CheckDataType", msg.Data());
1162}
1163
4e5c5506 1164//______________________________________________________________________________
1165void AliAnalysisAlien::EnablePackage(const char *package)
1166{
1167// Enables a par file supposed to exist in the current directory.
1168 TString pkg(package);
1169 pkg.ReplaceAll(".par", "");
1170 pkg += ".par";
1171 if (gSystem->AccessPathName(pkg)) {
1172 Error("EnablePackage", "Package %s not found", pkg.Data());
1173 return;
1174 }
1175 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1176 Info("EnablePackage", "AliEn plugin will use .par packages");
1177 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1178 if (!fPackages) {
1179 fPackages = new TObjArray();
1180 fPackages->SetOwner();
1181 }
1182 fPackages->Add(new TObjString(pkg));
1183}
1184
d2a409b2 1185//______________________________________________________________________________
1186const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1187{
1188// Get job status for all jobs with jobid>jobidstart.
1189 static char mstatus[20];
1190 mstatus[0] = '\0';
1191 nrunning = 0;
1192 nwaiting = 0;
1193 nerror = 0;
1194 ndone = 0;
1195 TGridJobStatusList *list = gGrid->Ps("");
1196 if (!list) return mstatus;
1197 Int_t nentries = list->GetSize();
1198 TGridJobStatus *status;
1199 Int_t pid;
1200 for (Int_t ijob=0; ijob<nentries; ijob++) {
1201 status = (TGridJobStatus *)list->At(ijob);
1202 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1203 if (pid<jobidstart) continue;
1204 if (pid == lastid) {
1205 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1206 }
1207 switch (status->GetStatus()) {
1208 case TGridJobStatus::kWAITING:
1209 nwaiting++; break;
1210 case TGridJobStatus::kRUNNING:
1211 nrunning++; break;
1212 case TGridJobStatus::kABORTED:
1213 case TGridJobStatus::kFAIL:
1214 case TGridJobStatus::kUNKNOWN:
1215 nerror++; break;
1216 case TGridJobStatus::kDONE:
1217 ndone++;
1218 }
1219 }
1220 list->Delete();
1221 delete list;
1222 return mstatus;
1223}
1224
c57f56b7 1225//______________________________________________________________________________
1226Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1227{
1228// Returns true if file is a collection. Functionality duplicated from
1229// TAlien::Type() because we don't want to directly depend on TAlien.
1230 if (!gGrid) {
1231 Error("IsCollection", "No connection to grid");
1232 return kFALSE;
1233 }
1234 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1235 if (!res) return kFALSE;
1236 const char* typeStr = res->GetKey(0, "type");
1237 if (!typeStr || !strlen(typeStr)) return kFALSE;
1238 if (!strcmp(typeStr, "collection")) return kTRUE;
1239 delete res;
1240 return kFALSE;
1241}
1242
fe2d7fc2 1243//______________________________________________________________________________
1244Bool_t AliAnalysisAlien::IsSingleOutput() const
1245{
1246// Check if single-ouput option is on.
1247 return (!fOutputSingle.IsNull());
1248}
1249
16a4353c 1250//______________________________________________________________________________
1251void AliAnalysisAlien::Print(Option_t *) const
1252{
1253// Print current plugin settings.
84fcd93f 1254 printf("### AliEn analysis plugin current settings ###\n");
1255 printf("= Production mode:______________________________ %d\n", fProductionMode);
1256 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1257 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1258 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1259 if (fUser.Length())
84fcd93f 1260 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1261 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1262 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1263 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1264 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1265 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1266 if (fRunNumbers.Length())
84fcd93f 1267 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1268 if (fRunRange[0])
5fce53f4 1269 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1270 if (!fRunRange[0] && !fRunNumbers.Length()) {
1271 TIter next(fInputFiles);
1272 TObject *obj;
1273 TString list;
1274 while ((obj=next())) list += obj->GetName();
84fcd93f 1275 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1276 }
1277 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1278 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1279 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1280 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1281 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1282 printf("=====================================================================\n");
1283 printf("= Job price: ___________________________________ %d\n", fPrice);
1284 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1285 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1286 if (fMaxInitFailed>0)
84fcd93f 1287 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1288 if (fMasterResubmitThreshold>0)
84fcd93f 1289 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1290 if (fNrunsPerMaster>0)
84fcd93f 1291 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1292 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
b5b9dee8 1293 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1294 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
16a4353c 1295 if (fArguments.Length())
84fcd93f 1296 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
631c0b05 1297 if (fExecutableArgs.Length())
1298 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
84fcd93f 1299 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1300 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1301 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1302 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1303 if (fDatasetName)
84fcd93f 1304 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1305 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1306 if (fIncludePath.Data())
84fcd93f 1307 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1308 if (fCloseSE.Length())
84fcd93f 1309 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1310 if (fFriendChainName.Length())
84fcd93f 1311 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1312 if (fPackages) {
1313 TIter next(fPackages);
1314 TObject *obj;
1315 TString list;
1316 while ((obj=next())) list += obj->GetName();
84fcd93f 1317 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1318 }
1319}
1320
c57f56b7 1321//______________________________________________________________________________
1322void AliAnalysisAlien::SetDefaults()
1323{
1324// Set default values for everything. What cannot be filled will be left empty.
1325 if (fGridJDL) delete fGridJDL;
1326 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1327 fPrice = 1;
1328 fTTL = 30000;
1329 fSplitMaxInputFileNumber = 100;
1330 fMaxInitFailed = 0;
1331 fMasterResubmitThreshold = 0;
bb885a9e 1332 fNtestFiles = 10;
a8739e8a 1333 fRunRange[0] = 0;
1334 fRunRange[1] = 0;
319593fb 1335 fNrunsPerMaster = 1;
16a4353c 1336 fMaxMergeFiles = 100;
c57f56b7 1337 fRunNumbers = "";
1338 fExecutable = "analysis.sh";
0a1c1f7f 1339 fExecutableCommand = "root -b -q";
c57f56b7 1340 fArguments = "";
631c0b05 1341 fExecutableArgs = "";
c57f56b7 1342 fAnalysisMacro = "myAnalysis.C";
1343 fAnalysisSource = "";
1344 fAdditionalLibs = "";
1345 fSplitMode = "se";
1346 fAPIVersion = "";
1347 fROOTVersion = "";
1348 fAliROOTVersion = "";
1349 fUser = ""; // Your alien user name
1350 fGridWorkingDir = "";
1351 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1352 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1353 fFriendChainName = "";
c57f56b7 1354 fGridOutputDir = "output";
1355 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1356 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1357 fInputFormat = "xml-single";
1358 fJDLName = "analysis.jdl";
c6cb3634 1359 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1360 fMergeExcludes = "";
c57f56b7 1361}
1362
1363//______________________________________________________________________________
1364Bool_t AliAnalysisAlien::MergeOutputs()
1365{
1366// Merge analysis outputs existing in the AliEn space.
1367 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1368 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1369 if (!Connect()) {
1370 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1371 return kFALSE;
1372 }
1373 // Get the output path
d2a409b2 1374 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1375 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1376 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1377 return kFALSE;
1378 }
1379 if (!fOutputFiles.Length()) {
1380 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1381 return kFALSE;
1382 }
1383 TObjArray *list = fOutputFiles.Tokenize(" ");
1384 TIter next(list);
1385 TObjString *str;
1386 TString command;
1387 TString output_file;
16a4353c 1388 TString output_chunk;
1389 TString previous_chunk;
1390 Int_t count_chunk = 0;
1391 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1392 Bool_t merged = kTRUE;
1393 while((str=(TObjString*)next())) {
1394 output_file = str->GetString();
1395 Int_t index = output_file.Index("@");
1396 if (index > 0) output_file.Remove(index);
319593fb 1397 // Skip already merged outputs
1398 if (!gSystem->AccessPathName(output_file)) {
1399 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1400 continue;
1401 }
bb885a9e 1402 if (fMergeExcludes.Length() &&
1403 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1404 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1405 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1406 printf("command: %s\n", command.Data());
c57f56b7 1407 TGridResult *res = gGrid->Command(command);
1408 if (!res) continue;
1409 TFileMerger *fm = 0;
1410 TIter nextmap(res);
ab254fd1 1411 TMap *map = 0;
16a4353c 1412 previous_chunk = "";
1413 count_chunk = 0;
319593fb 1414 // Check if there is a merge operation to resume
1415 output_chunk = output_file;
1416 output_chunk.ReplaceAll(".root", "_*.root");
1417 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1418 while (1) {
1419 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1420 if (!map) {
1421 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1422 delete res;
1423 return kFALSE;
1424 }
1425 output_chunk = output_file;
1426 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1427 printf("%s\n", output_chunk.Data());
319593fb 1428 count_chunk++;
1429 if (gSystem->AccessPathName(output_chunk)) continue;
1430 // Merged file with chunks up to <count_chunk> found
84fcd93f 1431 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1432 previous_chunk = output_chunk;
1433 break;
1434 }
1435 }
16a4353c 1436 count_zero = fMaxMergeFiles;
c57f56b7 1437 while ((map=(TMap*)nextmap())) {
16a4353c 1438 // Loop 'find' results and get next LFN
1439 if (count_zero == fMaxMergeFiles) {
1440 // First file in chunk - create file merger and add previous chunk if any.
1441 fm = new TFileMerger(kFALSE);
1442 fm->SetFastMethod(kTRUE);
1443 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1444 output_chunk = output_file;
1445 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1446 }
1447 // If last file found, put merged results in the output file
1448 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1449 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1450 if (!objs || !objs->GetString().Length()) {
16a4353c 1451 // Nothing found - skip this output
c57f56b7 1452 delete res;
16a4353c 1453 delete fm;
1454 break;
1455 }
1456 // Add file to be merged and decrement chunk counter.
1457 fm->AddFile(objs->GetString());
1458 count_zero--;
1459 if (count_zero==0 || map == res->Last()) {
1460 fm->OutputFile(output_chunk);
1461 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1462 // Nothing found - skip this output
1463 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1464 delete res;
1465 delete fm;
1466 break;
1467 }
1468 // Merge the outputs, then go to next chunk
1469 if (!fm->Merge()) {
1470 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1471 delete res;
1472 delete fm;
1473 merged = kFALSE;
1474 break;
1475 } else {
1476 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1477 gSystem->Unlink(previous_chunk);
16a4353c 1478 }
1479 if (map == res->Last()) {
1480 delete res;
1481 delete fm;
1482 break;
1483 }
1484 count_chunk++;
1485 count_zero = fMaxMergeFiles;
1486 previous_chunk = output_chunk;
c57f56b7 1487 }
c57f56b7 1488 }
c57f56b7 1489 }
1490 if (!merged) {
1491 Error("MergeOutputs", "Terminate() will NOT be executed");
1492 }
1493 return merged;
1494}
1495
bb885a9e 1496//______________________________________________________________________________
1497void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1498{
1499// Use the output files connected to output containers from the analysis manager
1500// rather than the files defined by SetOutputFiles
1501 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1502 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1503 analysis manager");
1504 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1505}
1506
5513444a 1507//______________________________________________________________________________
1508Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1509{
1510// Start remote grid analysis.
1511
43da816a 1512 // Check if output files have to be taken from the analysis manager
1513 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1514 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1515 if (!mgr || !mgr->IsInitialized()) {
1516 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1517 return kFALSE;
43da816a 1518 }
1519 fOutputFiles = "";
1520 TIter next(mgr->GetOutputs());
1521 AliAnalysisDataContainer *output;
1522 while ((output=(AliAnalysisDataContainer*)next())) {
1523 const char *filename = output->GetFileName();
1524 if (!(strcmp(filename, "default"))) {
1525 if (!mgr->GetOutputEventHandler()) continue;
1526 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1527 }
84fcd93f 1528 if (fOutputFiles.Contains(filename)) continue;
43da816a 1529 if (fOutputFiles.Length()) fOutputFiles += " ";
1530 fOutputFiles += filename;
1531 }
c07b9ce2 1532 // Add extra files registered to the analysis manager
1533 if (mgr->GetExtraFiles().Length()) {
1534 if (fOutputFiles.Length()) fOutputFiles += " ";
1535 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1536 }
43da816a 1537 }
f7b1cbc2 1538// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1539 if (TestBit(AliAnalysisGrid::kOffline)) {
1540 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1541 \n there nor any job run. You can revise the JDL and analysis \
1542 \n macro then run the same in \"submit\" mode.");
1543 } else if (TestBit(AliAnalysisGrid::kTest)) {
1544 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1545 \n dataset.");
1546 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1547 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1548 \n space and job submitted.");
1549 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1550 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1551 return kTRUE;
c57f56b7 1552 } else {
1553 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1554 }
1555
1556 if (!Connect()) {
1557 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1558 return kFALSE;
16a4353c 1559 }
1560 Print();
c57f56b7 1561 if (!CheckInputData()) {
1562 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1563 return kFALSE;
c57f56b7 1564 }
1565 CreateDataset(fDataPattern);
1566 WriteAnalysisFile();
1567 WriteAnalysisMacro();
1568 WriteExecutable();
1569 WriteValidationScript();
5513444a 1570 if (!CreateJDL()) return kFALSE;
1571 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1572 if (TestBit(AliAnalysisGrid::kTest)) {
1573 // Locally testing the analysis
1574 Info("StartAnalysis", "\n_______________________________________________________________________ \
1575 \n Running analysis script in a daughter shell as on a worker node \
1576 \n_______________________________________________________________________");
1577 TObjArray *list = fOutputFiles.Tokenize(" ");
1578 TIter next(list);
1579 TObjString *str;
1580 TString output_file;
1581 while((str=(TObjString*)next())) {
1582 output_file = str->GetString();
1583 Int_t index = output_file.Index("@");
1584 if (index > 0) output_file.Remove(index);
43da816a 1585 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1586 }
1587 delete list;
1588 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1589 TString validationScript = fExecutable;
1590 validationScript.ReplaceAll(".sh", "_validation.sh");
1591 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1592// gSystem->Exec("cat stdout");
5513444a 1593 return kFALSE;
c57f56b7 1594 }
5513444a 1595 // Check if submitting is managed by LPM manager
a3e84053 1596 if (fProductionMode) {
5513444a 1597 TString prodfile = fJDLName;
1598 prodfile.ReplaceAll(".jdl", ".prod");
1599 WriteProductionFile(prodfile);
1600 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1601 return kFALSE;
1602 }
a8739e8a 1603 // Submit AliEn job(s)
d2a409b2 1604 gGrid->Cd(fGridOutputDir);
a8739e8a 1605 TGridResult *res;
c57f56b7 1606 TString jobID = "";
d2a409b2 1607 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1608 // Submit a given xml or a set of runs
a8739e8a 1609 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1610 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1611 if (res) {
1612 const char *cjobId = res->GetKey(0,"jobId");
1613 if (!cjobId) {
1614 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1615 return kFALSE;
a8739e8a 1616 } else {
1617 Info("StartAnalysis", "\n_______________________________________________________________________ \
1618 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1619 \n_______________________________________________________________________",
1620 fJDLName.Data(), cjobId);
1621 jobID = cjobId;
1622 }
1623 delete res;
1624 }
1625 } else {
d2a409b2 1626 // Submit for a range of enumeration of runs.
1627 Submit();
c57f56b7 1628 }
a8739e8a 1629
c57f56b7 1630 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1631 \n You may exit at any time and terminate the job later using the option <terminate> \
1632 \n ##################################################################################", jobID.Data());
bb885a9e 1633 gSystem->Exec("aliensh");
5513444a 1634 return kTRUE;
c57f56b7 1635}
1636
d2a409b2 1637//______________________________________________________________________________
1638void AliAnalysisAlien::Submit()
1639{
1640// Submit all master jobs.
1641 Int_t nmasterjobs = fInputFiles->GetEntries();
1642 Long_t tshoot = gSystem->Now();
1643 if (!fNsubmitted) SubmitNext();
1644 while (fNsubmitted < nmasterjobs) {
1645 Long_t now = gSystem->Now();
1646 if ((now-tshoot)>30000) {
1647 tshoot = now;
1648 SubmitNext();
1649 }
1650 }
1651}
1652
1653//______________________________________________________________________________
1654void AliAnalysisAlien::SubmitNext()
1655{
1656// Submit next bunch of master jobs if the queue is free.
1657 static Bool_t iscalled = kFALSE;
1658 static Int_t firstmaster = 0;
1659 static Int_t lastmaster = 0;
1660 static Int_t npermaster = 0;
1661 if (iscalled) return;
1662 iscalled = kTRUE;
1663 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1664 Int_t ntosubmit = 0;
1665 TGridResult *res;
1666 TString jobID = "";
1667 if (!fNsubmitted) ntosubmit = 1;
1668 else {
1669 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1670 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1671 // If last master not split, just return
1672 if (status != "SPLIT") {iscalled = kFALSE; return;}
1673 // No more than 100 waiting jobs
1674 if (nwaiting>100) {iscalled = kFALSE; return;}
1675 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1676 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1677 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1678 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1679 }
1680 Int_t nmasterjobs = fInputFiles->GetEntries();
1681 for (Int_t i=0; i<ntosubmit; i++) {
1682 // Submit for a range of enumeration of runs.
1683 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1684 TString query;
cd11251e 1685 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
1686 runOutDir.ReplaceAll(".xml", "");
1687 if (fOutputToRunNo)
1688 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
1689 else
1690 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1691 printf("********* %s\n",query.Data());
d2a409b2 1692 res = gGrid->Command(query);
1693 if (res) {
98ca124f 1694 TString cjobId1 = res->GetKey(0,"jobId");
1695 if (!cjobId1.Length()) {
d2a409b2 1696 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1697 iscalled = kFALSE;
1698 return;
1699 } else {
1700 Info("StartAnalysis", "\n_______________________________________________________________________ \
1701 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1702 \n_______________________________________________________________________",
98ca124f 1703 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1704 jobID += cjobId1;
1705 jobID += " ";
98ca124f 1706 lastmaster = cjobId1.Atoi();
d2a409b2 1707 if (!firstmaster) firstmaster = lastmaster;
1708 fNsubmitted++;
1709 }
1710 delete res;
1711 }
1712 }
1713 iscalled = kFALSE;
1714}
1715
c57f56b7 1716//______________________________________________________________________________
1717void AliAnalysisAlien::WriteAnalysisFile()
1718{
f10e8481 1719// Write current analysis manager into the file <analysisFile>
1720 TString analysisFile = fExecutable;
1721 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1722 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1723 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1724 if (!mgr || !mgr->IsInitialized()) {
1725 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1726 return;
1727 }
1728 // Check analysis type
1729 TObject *handler;
1730 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1731 handler = (TObject*)mgr->GetInputEventHandler();
1732 if (handler) {
1733 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1734 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1735 }
1736 TDirectory *cdir = gDirectory;
f10e8481 1737 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1738 if (file) {
fe2d7fc2 1739 // Skip task Terminate calls for the grid job
1740 mgr->SetSkipTerminate(kTRUE);
1741 // Unless merging makes no sense
1742 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
c57f56b7 1743 mgr->Write();
1744 delete file;
fe2d7fc2 1745 // Enable termination for local jobs
1746 mgr->SetSkipTerminate(kFALSE);
c57f56b7 1747 }
1748 if (cdir) cdir->cd();
f10e8481 1749 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1750 }
1751 Bool_t copy = kTRUE;
1752 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1753 if (copy) {
1754 CdWork();
1755 TString workdir = gGrid->GetHomeDirectory();
1756 workdir += fGridWorkingDir;
f10e8481 1757 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1758 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1759 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1760 }
1761}
1762
1763//______________________________________________________________________________
1764void AliAnalysisAlien::WriteAnalysisMacro()
1765{
1766// Write the analysis macro that will steer the analysis in grid mode.
1767 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1768 ofstream out;
1769 out.open(fAnalysisMacro.Data(), ios::out);
1770 if (!out.good()) {
1771 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1772 return;
1773 }
1774 TString func = fAnalysisMacro;
1775 TString type = "ESD";
1776 TString comment = "// Analysis using ";
1777 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1778 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1779 type = "AOD";
1780 comment += "AOD";
1781 }
0df6ccf2 1782 if (type!="AOD" && fFriendChainName!="") {
1783 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1784 return;
1785 }
c57f56b7 1786 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1787 else comment += " data";
1788 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1789 func.ReplaceAll(".C", "");
1790 out << "void " << func.Data() << "()" << endl;
1791 out << "{" << endl;
1792 out << comment.Data() << endl;
1793 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1794 out << " TStopwatch timer;" << endl;
1795 out << " timer.Start();" << endl << endl;
c57f56b7 1796 out << "// load base root libraries" << endl;
1797 out << " gSystem->Load(\"libTree\");" << endl;
1798 out << " gSystem->Load(\"libGeom\");" << endl;
1799 out << " gSystem->Load(\"libVMC\");" << endl;
1800 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 1801 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
d5c6455a 1802 if (fAdditionalRootLibs.Length()) {
1803 // in principle libtree /lib geom libvmc etc. can go into this list, too
1804 out << "// Add aditional libraries" << endl;
1805 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
1806 TIter next(list);
1807 TObjString *str;
1808 while((str=(TObjString*)next())) {
1809 if (str->GetString().Contains(".so"))
1810 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1811 }
1812 if (list) delete list;
1813 }
57377eb5 1814 out << "// Load analysis framework libraries" << endl;
d5c6455a 1815
1816
4e5c5506 1817 if (!fPackages) {
4e5c5506 1818 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1819 out << " gSystem->Load(\"libESD\");" << endl;
1820 out << " gSystem->Load(\"libAOD\");" << endl;
1821 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1822 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1823 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1824 } else {
4e5c5506 1825 TIter next(fPackages);
1826 TObject *obj;
57377eb5 1827 TString pkgname;
1828 Bool_t hasSTEERBase = kFALSE;
1829 Bool_t hasESD = kFALSE;
1830 Bool_t hasAOD = kFALSE;
1831 Bool_t hasANALYSIS = kFALSE;
1832 Bool_t hasANALYSISalice = kFALSE;
1833 Bool_t hasCORRFW = kFALSE;
1834 while ((obj=next())) {
1835 pkgname = obj->GetName();
4478e6f1 1836 if (pkgname == "STEERBase" ||
1837 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1838 if (pkgname == "ESD" ||
1839 pkgname == "ESD.par") hasESD = kTRUE;
1840 if (pkgname == "AOD" ||
1841 pkgname == "AOD.par") hasAOD = kTRUE;
1842 if (pkgname == "ANALYSIS" ||
1843 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1844 if (pkgname == "ANALYSISalice" ||
1845 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1846 if (pkgname == "CORRFW" ||
1847 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1848 }
1849 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1850 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1851 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1852 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1853 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1854 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1855 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1856 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1857 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1858 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1859 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1860 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1861 out << "// Compile other par packages" << endl;
1862 next.Reset();
fcc9bb6f 1863 while ((obj=next())) {
1864 pkgname = obj->GetName();
4478e6f1 1865 if (pkgname == "STEERBase" ||
1866 pkgname == "STEERBase.par" ||
1867 pkgname == "ESD" ||
1868 pkgname == "ESD.par" ||
1869 pkgname == "AOD" ||
1870 pkgname == "AOD.par" ||
1871 pkgname == "ANALYSIS" ||
1872 pkgname == "ANALYSIS.par" ||
1873 pkgname == "ANALYSISalice" ||
1874 pkgname == "ANALYSISalice.par" ||
1875 pkgname == "CORRFW" ||
1876 pkgname == "CORRFW.par") continue;
4e5c5506 1877 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1878 }
4e5c5506 1879 }
43da816a 1880 out << "// include path" << endl;
1881 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1882 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1883 if (fAdditionalLibs.Length()) {
1884 out << "// Add aditional AliRoot libraries" << endl;
1885 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1886 TIter next(list);
1887 TObjString *str;
1888 while((str=(TObjString*)next())) {
1889 if (str->GetString().Contains(".so"))
1890 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1891 }
1892 if (list) delete list;
1893 }
1894 out << endl;
c57f56b7 1895 out << "// analysis source to be compiled at runtime (if any)" << endl;
1896 if (fAnalysisSource.Length()) {
1897 TObjArray *list = fAnalysisSource.Tokenize(" ");
1898 TIter next(list);
1899 TObjString *str;
1900 while((str=(TObjString*)next())) {
1901 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1902 }
1903 if (list) delete list;
1904 }
1905 out << endl;
1906 out << "// connect to AliEn and make the chain" << endl;
1907 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1908 if (IsUsingTags()) {
1909 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1910 } else {
76535c98 1911 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1912 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1913 } else {
1914 out << " // Check if the macro to create the chain was provided" << endl;
1915 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1916 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1917 out << " return;" << endl;
1918 out << " }" << endl;
1919 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1920 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1921 }
c57f56b7 1922 }
1923 out << "// read the analysis manager from file" << endl;
f10e8481 1924 TString analysisFile = fExecutable;
1925 analysisFile.ReplaceAll(".sh", ".root");
1926 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1927 out << " if (!file) return;" << endl;
1928 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1929 out << " AliAnalysisManager *mgr = 0;" << endl;
1930 out << " TKey *key;" << endl;
1931 out << " while ((key=(TKey*)nextkey())) {" << endl;
1932 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1933 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1934 out << " };" << endl;
1935 out << " if (!mgr) {" << endl;
f10e8481 1936 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1937 out << " return;" << endl;
1938 out << " }" << endl << endl;
1939 out << " mgr->PrintStatus();" << endl;
1940 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1941 out << " timer.Stop();" << endl;
1942 out << " timer.Print();" << endl;
c57f56b7 1943 out << "}" << endl << endl;
1944 if (IsUsingTags()) {
1945 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1946 out << "{" << endl;
1947 out << "// Create a chain using tags from the xml file." << endl;
1948 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1949 out << " if (!coll) {" << endl;
1950 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1951 out << " return NULL;" << endl;
1952 out << " }" << endl;
1953 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1954 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1955 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1956 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1957 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1958 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1959 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1960 out << " // Check if the cuts configuration file was provided" << endl;
1961 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1962 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1963 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1964 out << " }" << endl;
0df6ccf2 1965 if (fFriendChainName=="") {
1966 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1967 } else {
1968 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1969 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1970 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1971 }
c57f56b7 1972 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1973 out << " chain->ls();" << endl;
1974 out << " return chain;" << endl;
fcc9bb6f 1975 out << "}" << endl << endl;
c57f56b7 1976 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1977 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1978 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1979 msg += " AliLHCTagCuts *lhcCuts,\n";
1980 msg += " AliDetectorTagCuts *detCuts,\n";
1981 msg += " AliEventTagCuts *evCuts)";
1982 Info("WriteAnalysisMacro", msg.Data());
1983 }
0df6ccf2 1984 }
1985 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1986 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1987 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1988 out << "{" << endl;
1989 out << "// Create a chain using url's from xml file" << endl;
1990 out << " TString treename = type;" << endl;
1991 out << " treename.ToLower();" << endl;
1992 out << " treename += \"Tree\";" << endl;
e02fee64 1993 out << " printf(\"***************************************\\n\");" << endl;
1994 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1995 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1996 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1997 out << " if (!coll) {" << endl;
1998 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1999 out << " return NULL;" << endl;
2000 out << " }" << endl;
2001 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 2002 if(fFriendChainName!="") {
2003 out << " TChain *chainFriend = new TChain(treename);" << endl;
2004 }
c57f56b7 2005 out << " coll->Reset();" << endl;
0df6ccf2 2006 out << " while (coll->Next()) {" << endl;
2007 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
2008 if(fFriendChainName!="") {
2009 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
2010 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2011 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
2012 out << " chainFriend->Add(fileFriend.Data());" << endl;
2013 }
2014 out << " }" << endl;
c57f56b7 2015 out << " if (!chain->GetNtrees()) {" << endl;
2016 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
2017 out << " return NULL;" << endl;
2018 out << " }" << endl;
0df6ccf2 2019 if(fFriendChainName!="") {
2020 out << " chain->AddFriend(chainFriend);" << endl;
2021 }
c57f56b7 2022 out << " return chain;" << endl;
fcc9bb6f 2023 out << "}" << endl << endl;
c57f56b7 2024 }
4e5c5506 2025 if (fPackages) {
fcc9bb6f 2026 out <<"//________________________________________________________________________________" << endl;
4e5c5506 2027 out << "Bool_t SetupPar(const char *package) {" << endl;
2028 out << "// Compile the package and set it up." << endl;
2029 out << " TString pkgdir = package;" << endl;
2030 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 2031 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 2032 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2033 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2034 out << " // Check for BUILD.sh and execute" << endl;
2035 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2036 out << " printf(\"*******************************\\n\");" << endl;
2037 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2038 out << " printf(\"*******************************\\n\");" << endl;
2039 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 2040 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 2041 out << " gSystem->ChangeDirectory(cdir);" << endl;
2042 out << " return kFALSE;" << endl;
2043 out << " }" << endl;
2044 out << " } else {" << endl;
fcc9bb6f 2045 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 2046 out << " gSystem->ChangeDirectory(cdir);" << endl;
2047 out << " return kFALSE;" << endl;
2048 out << " }" << endl;
2049 out << " // Check for SETUP.C and execute" << endl;
2050 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2051 out << " printf(\"*******************************\\n\");" << endl;
2052 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2053 out << " printf(\"*******************************\\n\");" << endl;
2054 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2055 out << " } else {" << endl;
fcc9bb6f 2056 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 2057 out << " gSystem->ChangeDirectory(cdir);" << endl;
2058 out << " return kFALSE;" << endl;
2059 out << " }" << endl;
2060 out << " // Restore original workdir" << endl;
2061 out << " gSystem->ChangeDirectory(cdir);" << endl;
2062 out << " return kTRUE;" << endl;
2063 out << "}" << endl;
2064 }
c57f56b7 2065 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2066 }
2067 Bool_t copy = kTRUE;
2068 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2069 if (copy) {
2070 CdWork();
2071 TString workdir = gGrid->GetHomeDirectory();
2072 workdir += fGridWorkingDir;
2073 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2074 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2075 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2076 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2077 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2078 }
2079 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2080 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2081 }
2082}
2083
2084//______________________________________________________________________________
2085void AliAnalysisAlien::WriteExecutable()
2086{
2087// Generate the alien executable script.
2088 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2089 ofstream out;
2090 out.open(fExecutable.Data(), ios::out);
2091 if (out.bad()) {
5513444a 2092 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2093 return;
2094 }
2095 out << "#!/bin/bash" << endl;
c57f56b7 2096 out << "echo \"=========================================\"" << endl;
2097 out << "echo \"############## PATH : ##############\"" << endl;
2098 out << "echo $PATH" << endl;
2099 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2100 out << "echo $LD_LIBRARY_PATH" << endl;
2101 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2102 out << "echo $ROOTSYS" << endl;
2103 out << "echo \"############## which root : ##############\"" << endl;
2104 out << "which root" << endl;
2105 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2106 out << "echo $ALICE_ROOT" << endl;
2107 out << "echo \"############## which aliroot : ##############\"" << endl;
2108 out << "which aliroot" << endl;
9c5ddadc 2109 out << "echo \"############## system limits : ##############\"" << endl;
2110 out << "ulimit -a" << endl;
2111 out << "echo \"############## memory : ##############\"" << endl;
2112 out << "free -m" << endl;
c57f56b7 2113 out << "echo \"=========================================\"" << endl << endl;
74a53467 2114 // Make sure we can properly compile par files
2115 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2116 out << fExecutableCommand << " ";
631c0b05 2117 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
9c5ddadc 2118 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2119 out << "echo \"############## memory after: ##############\"" << endl;
2120 out << "free -m" << endl;
2121 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2122 out << "dmesg | tail -n 10" << endl;
c57f56b7 2123 }
2124 Bool_t copy = kTRUE;
2125 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2126 if (copy) {
2127 CdWork();
2128 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2129 TString bindir = Form("%s/bin", workdir.Data());
2130 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2131 workdir += fGridWorkingDir;
2132 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2133 if (FileExists(executable)) gGrid->Rm(executable);
2134 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2135 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2136 }
2137}
2138
2139//______________________________________________________________________________
5513444a 2140void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2141{
2142// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2143// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2144// Next lines: full_path_to_dataset XXX (XXX is a string)
2145// To submit, one has to: submit jdl XXX for all lines
2146 ofstream out;
2147 out.open(filename, ios::out);
2148 if (out.bad()) {
2149 Error("WriteProductionFile", "Bad file name: %s", filename);
2150 return;
2151 }
2152 TString workdir = gGrid->GetHomeDirectory();
2153 workdir += fGridWorkingDir;
f5e8c702 2154 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2155 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2156 out << locjdl << " " << njobspermaster << endl;
5513444a 2157 Int_t nmasterjobs = fInputFiles->GetEntries();
2158 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2159 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2160 }
2161 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2162 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2163}
2164
2165//______________________________________________________________________________
c57f56b7 2166void AliAnalysisAlien::WriteValidationScript()
2167{
2168// Generate the alien validation script.
2169 // Generate the validation script
2170 TObjString *os;
0d5d317c 2171 TString validationScript = fExecutable;
2172 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2173 if (!Connect()) {
2174 Error("WriteValidationScript", "Alien connection required");
2175 return;
2176 }
2177 TString out_stream = "";
2178 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2179 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2180 ofstream out;
0d5d317c 2181 out.open(validationScript, ios::out);
c57f56b7 2182 out << "#!/bin/bash" << endl;
2183 out << "##################################################" << endl;
2184 out << "validateout=`dirname $0`" << endl;
2185 out << "validatetime=`date`" << endl;
2186 out << "validated=\"0\";" << endl;
2187 out << "error=0" << endl;
2188 out << "if [ -z $validateout ]" << endl;
2189 out << "then" << endl;
2190 out << " validateout=\".\"" << endl;
2191 out << "fi" << endl << endl;
2192 out << "cd $validateout;" << endl;
2193 out << "validateworkdir=`pwd`;" << endl << endl;
2194 out << "echo \"*******************************************************\"" << out_stream << endl;
2195 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2196 out << "" << endl;
2197 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2198 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2199 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2200 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2201 out << "ls -la ./" << out_stream << endl;
2202 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2203 out << "##################################################" << endl;
ebec370a 2204
2205 out << "" << endl;
2206 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2207 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2208 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2209 out << "" << endl;
2210
2211 out << "if [ ! -f stderr ] ; then" << endl;
2212 out << " error=1" << endl;
2213 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2214 out << " echo \"Error = $error\" " << out_stream << endl;
2215 out << "fi" << endl;
2216
2217 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2218 out << " error=1" << endl;
2219 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2220 out << " echo \"$parArch\" " << out_stream << endl;
2221 out << " echo \"Error = $error\" " << out_stream << endl;
2222 out << "fi" << endl;
2223
2224 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2225 out << " error=1" << endl;
2226 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2227 out << " echo \"$segViol\" " << out_stream << endl;
2228 out << " echo \"Error = $error\" " << out_stream << endl;
2229 out << "fi" << endl;
2230
2231 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2232 out << " error=1" << endl;
2233 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2234 out << " echo \"$segFault\" " << out_stream << endl;
2235 out << " echo \"Error = $error\" " << out_stream << endl;
2236 out << "fi" << endl;
2237
2238 // Part dedicated to the specific analyses running into the train
2239
c57f56b7 2240 TObjArray *arr = fOutputFiles.Tokenize(" ");
2241 TIter next1(arr);
2242 TString output_file;
2243 while ((os=(TObjString*)next1())) {
2244 output_file = os->GetString();
2245 Int_t index = output_file.Index("@");
2246 if (index > 0) output_file.Remove(index);
2247 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2248 out << " error=1" << endl;
2249 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2250 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2251 out << "fi" << endl;
2252 }
2253 delete arr;
923e2ca5 2254 out << "if ! [ -f outputs_valid ] ; then" << endl;
2255 out << " error=1" << endl;
2256 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2257 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2258 out << "fi" << endl;
2259
c57f56b7 2260 out << "if [ $error = 0 ] ; then" << endl;
2261 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2262 out << "fi" << endl;
2263
2264 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2265 out << "echo \"*******************************************************\"" << out_stream << endl;
2266 out << "cd -" << endl;
2267 out << "exit $error" << endl;
2268 }
2269 Bool_t copy = kTRUE;
2270 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2271 if (copy) {
2272 CdWork();
2273 TString workdir = gGrid->GetHomeDirectory();
2274 workdir += fGridWorkingDir;
0d5d317c 2275 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2276 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2277 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2278 }
2279}