]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
D0 trigger update by Gaute
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
0a1c1f7f 59 fExecutableCommand(),
c57f56b7 60 fArguments(),
61 fAnalysisMacro(),
62 fAnalysisSource(),
63 fAdditionalLibs(),
64 fSplitMode(),
65 fAPIVersion(),
66 fROOTVersion(),
67 fAliROOTVersion(),
648174cf 68 fExternalPackages(),
c57f56b7 69 fUser(),
70 fGridWorkingDir(),
71 fGridDataDir(),
72 fDataPattern(),
73 fGridOutputDir(),
74 fOutputArchive(),
75 fOutputFiles(),
76 fInputFormat(),
e7c71df0 77 fDatasetName(),
c57f56b7 78 fJDLName(),
bb885a9e 79 fMergeExcludes(),
f965131e 80 fIncludePath(),
bb885a9e 81 fCloseSE(),
0df6ccf2 82 fFriendChainName(),
c6cb3634 83 fJobTag(),
648174cf 84 fOutputSingle(),
5fce53f4 85 fRunPrefix(),
4e5c5506 86 fInputFiles(0),
87 fPackages(0)
c57f56b7 88{
89// Dummy ctor.
90 SetDefaults();
91}
92
93//______________________________________________________________________________
94AliAnalysisAlien::AliAnalysisAlien(const char *name)
95 :AliAnalysisGrid(name),
96 fGridJDL(NULL),
97 fPrice(0),
98 fTTL(0),
99 fSplitMaxInputFileNumber(0),
100 fMaxInitFailed(0),
101 fMasterResubmitThreshold(0),
bb885a9e 102 fNtestFiles(0),
319593fb 103 fNrunsPerMaster(0),
16a4353c 104 fMaxMergeFiles(0),
d2a409b2 105 fNsubmitted(0),
a3e84053 106 fProductionMode(0),
c57f56b7 107 fRunNumbers(),
108 fExecutable(),
0a1c1f7f 109 fExecutableCommand(),
c57f56b7 110 fArguments(),
111 fAnalysisMacro(),
112 fAnalysisSource(),
113 fAdditionalLibs(),
114 fSplitMode(),
115 fAPIVersion(),
116 fROOTVersion(),
117 fAliROOTVersion(),
648174cf 118 fExternalPackages(),
c57f56b7 119 fUser(),
120 fGridWorkingDir(),
121 fGridDataDir(),
122 fDataPattern(),
123 fGridOutputDir(),
124 fOutputArchive(),
125 fOutputFiles(),
126 fInputFormat(),
e7c71df0 127 fDatasetName(),
c57f56b7 128 fJDLName(),
bb885a9e 129 fMergeExcludes(),
f965131e 130 fIncludePath(),
bb885a9e 131 fCloseSE(),
0df6ccf2 132 fFriendChainName(),
c6cb3634 133 fJobTag(),
648174cf 134 fOutputSingle(),
5fce53f4 135 fRunPrefix(),
4e5c5506 136 fInputFiles(0),
137 fPackages(0)
c57f56b7 138{
139// Default ctor.
140 SetDefaults();
141}
142
143//______________________________________________________________________________
144AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
145 :AliAnalysisGrid(other),
146 fGridJDL(NULL),
147 fPrice(other.fPrice),
148 fTTL(other.fTTL),
149 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
150 fMaxInitFailed(other.fMaxInitFailed),
151 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 152 fNtestFiles(other.fNtestFiles),
319593fb 153 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 154 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 155 fNsubmitted(other.fNsubmitted),
a3e84053 156 fProductionMode(other.fProductionMode),
c57f56b7 157 fRunNumbers(other.fRunNumbers),
158 fExecutable(other.fExecutable),
0a1c1f7f 159 fExecutableCommand(other.fExecutableCommand),
c57f56b7 160 fArguments(other.fArguments),
161 fAnalysisMacro(other.fAnalysisMacro),
162 fAnalysisSource(other.fAnalysisSource),
163 fAdditionalLibs(other.fAdditionalLibs),
164 fSplitMode(other.fSplitMode),
165 fAPIVersion(other.fAPIVersion),
166 fROOTVersion(other.fROOTVersion),
167 fAliROOTVersion(other.fAliROOTVersion),
648174cf 168 fExternalPackages(other.fExternalPackages),
c57f56b7 169 fUser(other.fUser),
170 fGridWorkingDir(other.fGridWorkingDir),
171 fGridDataDir(other.fGridDataDir),
172 fDataPattern(other.fDataPattern),
173 fGridOutputDir(other.fGridOutputDir),
174 fOutputArchive(other.fOutputArchive),
175 fOutputFiles(other.fOutputFiles),
176 fInputFormat(other.fInputFormat),
e7c71df0 177 fDatasetName(other.fDatasetName),
c57f56b7 178 fJDLName(other.fJDLName),
bb885a9e 179 fMergeExcludes(other.fMergeExcludes),
f965131e 180 fIncludePath(other.fIncludePath),
bb885a9e 181 fCloseSE(other.fCloseSE),
0df6ccf2 182 fFriendChainName(other.fFriendChainName),
c6cb3634 183 fJobTag(other.fJobTag),
648174cf 184 fOutputSingle(other.fOutputSingle),
5fce53f4 185 fRunPrefix(other.fRunPrefix),
4e5c5506 186 fInputFiles(0),
187 fPackages(0)
c57f56b7 188{
189// Copy ctor.
190 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 191 fRunRange[0] = other.fRunRange[0];
192 fRunRange[1] = other.fRunRange[1];
c57f56b7 193 if (other.fInputFiles) {
194 fInputFiles = new TObjArray();
195 TIter next(other.fInputFiles);
196 TObject *obj;
197 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
198 fInputFiles->SetOwner();
199 }
4e5c5506 200 if (other.fPackages) {
201 fPackages = new TObjArray();
202 TIter next(other.fPackages);
203 TObject *obj;
204 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
205 fPackages->SetOwner();
206 }
c57f56b7 207}
208
209//______________________________________________________________________________
210AliAnalysisAlien::~AliAnalysisAlien()
211{
212// Destructor.
213 if (fGridJDL) delete fGridJDL;
214 if (fInputFiles) delete fInputFiles;
4e5c5506 215 if (fPackages) delete fPackages;
c57f56b7 216}
217
218//______________________________________________________________________________
219AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
220{
221// Assignment.
222 if (this != &other) {
223 AliAnalysisGrid::operator=(other);
224 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
225 fPrice = other.fPrice;
226 fTTL = other.fTTL;
227 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
228 fMaxInitFailed = other.fMaxInitFailed;
229 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 230 fNtestFiles = other.fNtestFiles;
a3e84053 231 fNrunsPerMaster = other.fNrunsPerMaster;
232 fMaxMergeFiles = other.fMaxMergeFiles;
233 fNsubmitted = other.fNsubmitted;
234 fProductionMode = other.fProductionMode;
c57f56b7 235 fRunNumbers = other.fRunNumbers;
236 fExecutable = other.fExecutable;
0a1c1f7f 237 fExecutableCommand = other.fExecutableCommand;
c57f56b7 238 fArguments = other.fArguments;
239 fAnalysisMacro = other.fAnalysisMacro;
240 fAnalysisSource = other.fAnalysisSource;
241 fAdditionalLibs = other.fAdditionalLibs;
242 fSplitMode = other.fSplitMode;
243 fAPIVersion = other.fAPIVersion;
244 fROOTVersion = other.fROOTVersion;
245 fAliROOTVersion = other.fAliROOTVersion;
648174cf 246 fExternalPackages = other.fExternalPackages;
c57f56b7 247 fUser = other.fUser;
248 fGridWorkingDir = other.fGridWorkingDir;
249 fGridDataDir = other.fGridDataDir;
250 fDataPattern = other.fDataPattern;
251 fGridOutputDir = other.fGridOutputDir;
252 fOutputArchive = other.fOutputArchive;
253 fOutputFiles = other.fOutputFiles;
254 fInputFormat = other.fInputFormat;
e7c71df0 255 fDatasetName = other.fDatasetName;
c57f56b7 256 fJDLName = other.fJDLName;
bb885a9e 257 fMergeExcludes = other.fMergeExcludes;
f965131e 258 fIncludePath = other.fIncludePath;
bb885a9e 259 fCloseSE = other.fCloseSE;
0df6ccf2 260 fFriendChainName = other.fFriendChainName;
c6cb3634 261 fJobTag = other.fJobTag;
648174cf 262 fOutputSingle = other.fOutputSingle;
5fce53f4 263 fRunPrefix = other.fRunPrefix;
c57f56b7 264 if (other.fInputFiles) {
265 fInputFiles = new TObjArray();
266 TIter next(other.fInputFiles);
267 TObject *obj;
268 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
269 fInputFiles->SetOwner();
270 }
4e5c5506 271 if (other.fPackages) {
272 fPackages = new TObjArray();
273 TIter next(other.fPackages);
274 TObject *obj;
275 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
276 fPackages->SetOwner();
277 }
c57f56b7 278 }
279 return *this;
280}
281
f965131e 282//______________________________________________________________________________
283void AliAnalysisAlien::AddIncludePath(const char *path)
284{
285// Add include path in the remote analysis macro.
286 TString p(path);
287 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
288 else fIncludePath += Form("-I%s ", path);
289}
290
c57f56b7 291//______________________________________________________________________________
292void AliAnalysisAlien::AddRunNumber(Int_t run)
293{
294// Add a run number to the list of runs to be processed.
295 if (fRunNumbers.Length()) fRunNumbers += " ";
5fce53f4 296 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
c57f56b7 297}
298
ee75cfc3 299//______________________________________________________________________________
300void AliAnalysisAlien::AddRunNumber(const char* run)
301{
302// Add a run number to the list of runs to be processed.
303 if (fRunNumbers.Length()) fRunNumbers += " ";
304 fRunNumbers += run;
305}
306
c57f56b7 307//______________________________________________________________________________
308void AliAnalysisAlien::AddDataFile(const char *lfn)
309{
310// Adds a data file to the input to be analysed. The file should be a valid LFN
311// or point to an existing file in the alien workdir.
312 if (!fInputFiles) fInputFiles = new TObjArray();
313 fInputFiles->Add(new TObjString(lfn));
314}
648174cf 315
316//______________________________________________________________________________
317void AliAnalysisAlien::AddExternalPackage(const char *package)
318{
319// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
320 if (fExternalPackages) fExternalPackages += " ";
321 fExternalPackages += package;
322}
323
c57f56b7 324//______________________________________________________________________________
325Bool_t AliAnalysisAlien::Connect()
326{
327// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
328 if (gGrid && gGrid->IsConnected()) return kTRUE;
329 if (!gSystem->Getenv("alien_API_USER")) {
330 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
331 gSystem->Getenv("UID"));
332 return kFALSE;
333 }
334 if (!gGrid) {
335 Info("Connect", "Trying to connect to AliEn ...");
336 TGrid::Connect("alien://");
337 }
338 if (!gGrid || !gGrid->IsConnected()) {
339 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
340 return kFALSE;
341 }
342 fUser = gGrid->GetUser();
343 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
344 return kTRUE;
345}
346
347//______________________________________________________________________________
348void AliAnalysisAlien::CdWork()
349{
350// Check validity of alien workspace. Create directory if possible.
351 if (!Connect()) {
352 Error("CdWork", "Alien connection required");
353 return;
354 }
355 TString homedir = gGrid->GetHomeDirectory();
356 TString workdir = homedir + fGridWorkingDir;
923e2ca5 357 if (DirectoryExists(workdir)) {
358 gGrid->Cd(workdir);
359 return;
360 }
361 // Work directory not existing - create it
362 gGrid->Cd(homedir);
363 if (gGrid->Mkdir(workdir)) {
364 gGrid->Cd(fGridWorkingDir);
365 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
366 } else {
367 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
368 workdir.Data(), homedir.Data());
369 fGridWorkingDir = "";
370 }
c57f56b7 371}
372
373//______________________________________________________________________________
374Bool_t AliAnalysisAlien::CheckInputData()
375{
376// Check validity of input data. If necessary, create xml files.
a8739e8a 377 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 378 if (!fGridDataDir.Length()) {
379 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
380 return kFALSE;
381 }
382 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
383 return kTRUE;
c57f56b7 384 }
385 // Process declared files
386 Bool_t is_collection = kFALSE;
387 Bool_t is_xml = kFALSE;
388 Bool_t use_tags = kFALSE;
389 Bool_t checked = kFALSE;
390 CdWork();
391 TString file;
392 TString workdir = gGrid->GetHomeDirectory();
393 workdir += fGridWorkingDir;
394 if (fInputFiles) {
395 TObjString *objstr;
396 TIter next(fInputFiles);
397 while ((objstr=(TObjString*)next())) {
398 file = workdir;
399 file += "/";
400 file += objstr->GetString();
401 // Store full lfn path
402 if (FileExists(file)) objstr->SetString(file);
403 else {
404 file = objstr->GetName();
405 if (!FileExists(objstr->GetName())) {
406 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
407 objstr->GetName(), workdir.Data());
408 return kFALSE;
409 }
410 }
411 Bool_t iscoll, isxml, usetags;
412 CheckDataType(file, iscoll, isxml, usetags);
413 if (!checked) {
414 checked = kTRUE;
415 is_collection = iscoll;
416 is_xml = isxml;
417 use_tags = usetags;
418 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
419 } else {
420 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
421 Error("CheckInputData", "Some conflict was found in the types of inputs");
422 return kFALSE;
423 }
424 }
425 }
426 }
427 // Process requested run numbers
a8739e8a 428 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 429 // Check validity of alien data directory
430 if (!fGridDataDir.Length()) {
431 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
432 return kFALSE;
433 }
923e2ca5 434 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 435 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
436 return kFALSE;
437 }
438 if (is_collection) {
439 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
440 return kFALSE;
441 }
442
443 if (checked && !is_xml) {
444 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
445 return kFALSE;
446 }
447 // Check validity of run number(s)
448 TObjArray *arr;
449 TObjString *os;
319593fb 450 Int_t nruns = 0;
904f9f5f 451 TString schunk, schunk2;
c57f56b7 452 TString path;
453 if (!checked) {
454 checked = kTRUE;
455 use_tags = fDataPattern.Contains("tag");
456 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
457 }
458 if (use_tags != fDataPattern.Contains("tag")) {
459 Error("CheckInputData", "Cannot mix input files using/not using tags");
460 return kFALSE;
461 }
462 if (fRunNumbers.Length()) {
a8739e8a 463 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 464 arr = fRunNumbers.Tokenize(" ");
465 TIter next(arr);
466 while ((os=(TObjString*)next())) {
467 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 468 if (!DirectoryExists(path)) {
469 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 470 continue;
c57f56b7 471 }
472 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
473 TString msg = "\n##### file: ";
474 msg += path;
475 msg += " type: xml_collection;";
476 if (use_tags) msg += " using_tags: Yes";
477 else msg += " using_tags: No";
478 Info("CheckDataType", msg.Data());
319593fb 479 if (fNrunsPerMaster<2) {
d2a409b2 480 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 481 } else {
482 nruns++;
483 if (((nruns-1)%fNrunsPerMaster) == 0) {
484 schunk = os->GetString();
485 }
486 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
487 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 488 AddDataFile(schunk);
319593fb 489 }
c57f56b7 490 }
491 delete arr;
a8739e8a 492 } else {
493 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
494 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 495 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 496 if (!DirectoryExists(path)) {
497// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 498 continue;
499 }
5fce53f4 500 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
a8739e8a 501 TString msg = "\n##### file: ";
502 msg += path;
503 msg += " type: xml_collection;";
504 if (use_tags) msg += " using_tags: Yes";
505 else msg += " using_tags: No";
506 Info("CheckDataType", msg.Data());
319593fb 507 if (fNrunsPerMaster<2) {
5fce53f4 508 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
319593fb 509 } else {
510 nruns++;
511 if (((nruns-1)%fNrunsPerMaster) == 0) {
5fce53f4 512 schunk = Form("%s%d", fRunPrefix.Data(),irun);
319593fb 513 }
904f9f5f 514 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
319593fb 515 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
904f9f5f 516 schunk += schunk2;
d2a409b2 517 AddDataFile(schunk);
319593fb 518 }
a8739e8a 519 }
904f9f5f 520 if (!fInputFiles) {
521 schunk += schunk2;
522 AddDataFile(schunk);
523 }
c57f56b7 524 }
525 return kTRUE;
526}
527
528//______________________________________________________________________________
529Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
530{
531// Create dataset for the grid data directory + run number.
532 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
533 if (!Connect()) {
534 Error("CreateDataset", "Cannot create dataset with no grid connection");
535 return kFALSE;
536 }
537
538 // Cd workspace
539 CdWork();
540 TString workdir = gGrid->GetHomeDirectory();
541 workdir += fGridWorkingDir;
542
543 // Compose the 'find' command arguments
544 TString command;
545 TString options = "-x collection ";
bb885a9e 546 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 547 TString conditions = "";
548
549 TString file;
550 TString path;
319593fb 551 Int_t nruns = 0;
904f9f5f 552 TString schunk, schunk2;
ab254fd1 553 TGridCollection *cbase=0, *cadd=0;
d2a409b2 554 if (!fRunNumbers.Length() && !fRunRange[0]) {
555 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
556 // Make a single data collection from data directory.
557 path = fGridDataDir;
923e2ca5 558 if (!DirectoryExists(path)) {
d2a409b2 559 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
560 return kFALSE;
561 }
923e2ca5 562// CdWork();
d2a409b2 563 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
564 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 565 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 566 command = "find ";
567 command += options;
568 command += path;
569 command += " ";
570 command += pattern;
571 command += conditions;
84fcd93f 572 printf("command: %s\n", command.Data());
d2a409b2 573 TGridResult *res = gGrid->Command(command);
574 if (res) delete res;
575 // Write standard output to file
576 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
577 }
578 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
579 // Copy xml file to alien space
580 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
581 if (!FileExists(file)) {
582 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
583 return kFALSE;
584 }
585 // Update list of files to be processed.
586 }
587 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
588 return kTRUE;
589 }
c57f56b7 590 // Several runs
a8739e8a 591 if (fRunNumbers.Length()) {
592 TObjArray *arr = fRunNumbers.Tokenize(" ");
593 TObjString *os;
594 TIter next(arr);
595 while ((os=(TObjString*)next())) {
596 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 597 if (!DirectoryExists(path)) continue;
598// CdWork();
a8739e8a 599 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
600 else file = Form("%s.xml", os->GetString().Data());
319593fb 601 // If local collection file does not exist, create it via 'find' command.
602 if (gSystem->AccessPathName(file)) {
603 command = "find ";
604 command += options;
605 command += path;
606 command += pattern;
607 command += conditions;
608 TGridResult *res = gGrid->Command(command);
609 if (res) delete res;
610 // Write standard output to file
611 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
612 }
a8739e8a 613 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 614 // Check if there is one run per master job.
615 if (fNrunsPerMaster<2) {
616 if (FileExists(file)) {
617 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
618 continue;
619 }
620 // Copy xml file to alien space
621 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
622 if (!FileExists(file)) {
623 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
624 delete arr;
625 return kFALSE;
626 }
627 } else {
628 nruns++;
629 if (((nruns-1)%fNrunsPerMaster) == 0) {
630 schunk = os->GetString();
631 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
632 } else {
633 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 634 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 635 cbase->Add(cadd);
636 delete cadd;
637 }
638 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
639 continue;
640 }
641 schunk += Form("_%s.xml", os->GetString().Data());
642 if (FileExists(schunk)) {
643 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
644 continue;
645 }
84fcd93f 646 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 647 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 648 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 649 if (!FileExists(schunk)) {
650 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
651 delete arr;
652 return kFALSE;
653 }
654 }
a8739e8a 655 }
656 delete arr;
657 } else {
658 // Process a full run range.
659 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
5fce53f4 660 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
923e2ca5 661 if (!DirectoryExists(path)) continue;
662// CdWork();
a8739e8a 663 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
5fce53f4 664 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
319593fb 665 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 666 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
667// gGrid->Rm(file);
668 continue;
669 }
319593fb 670 // If local collection file does not exist, create it via 'find' command.
671 if (gSystem->AccessPathName(file)) {
672 command = "find ";
673 command += options;
674 command += path;
675 command += pattern;
676 command += conditions;
677 TGridResult *res = gGrid->Command(command);
678 if (res) delete res;
679 // Write standard output to file
680 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
681 }
a8739e8a 682 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 683 // Check if there is one run per master job.
684 if (fNrunsPerMaster<2) {
685 if (FileExists(file)) {
686 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
687 continue;
688 }
689 // Copy xml file to alien space
690 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
691 if (!FileExists(file)) {
692 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
693 return kFALSE;
694 }
695 } else {
696 nruns++;
95e5b448 697 // Check if the collection for the chunk exist locally.
698 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
699 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 700 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 701 if (((nruns-1)%fNrunsPerMaster) == 0) {
904f9f5f 702 schunk = Form("%s%d", fRunPrefix.Data(), irun);
319593fb 703 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
704 } else {
705 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
706 cbase->Add(cadd);
707 delete cadd;
708 }
904f9f5f 709 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
710 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
319593fb 711 continue;
712 }
904f9f5f 713 schunk = schunk2;
319593fb 714 if (FileExists(schunk)) {
715 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
716 continue;
717 }
84fcd93f 718 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 719 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 720 if (FileExists(schunk)) {
721 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
722 continue;
723 }
319593fb 724 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
725 if (!FileExists(schunk)) {
726 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
727 return kFALSE;
728 }
729 }
c57f56b7 730 }
a8739e8a 731 }
c57f56b7 732 return kTRUE;
733}
734
735//______________________________________________________________________________
736Bool_t AliAnalysisAlien::CreateJDL()
737{
738// Generate a JDL file according to current settings. The name of the file is
739// specified by fJDLName.
740 Bool_t error = kFALSE;
741 TObjArray *arr = 0;
742 Bool_t copy = kTRUE;
743 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
744 Bool_t generate = kTRUE;
745 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
746 if (!Connect()) {
747 Error("CreateJDL", "Alien connection required");
748 return kFALSE;
749 }
750 // Check validity of alien workspace
751 CdWork();
752 TString workdir = gGrid->GetHomeDirectory();
753 workdir += fGridWorkingDir;
754 if (generate) {
755 TObjString *os;
756 if (!fInputFiles) {
757 Error("CreateJDL()", "Define some input files for your analysis.");
758 error = kTRUE;
759 }
760 // Compose list of input files
761 // Check if output files were defined
762 if (!fOutputFiles.Length()) {
763 Error("CreateJDL", "You must define at least one output file");
764 error = kTRUE;
765 }
766 // Check if an output directory was defined and valid
767 if (!fGridOutputDir.Length()) {
768 Error("CreateJDL", "You must define AliEn output directory");
769 error = kTRUE;
770 } else {
d2a409b2 771 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 772 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 773 if (gGrid->Mkdir(fGridOutputDir)) {
774 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
775 } else {
776 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 777 // error = kTRUE;
c57f56b7 778 }
779 }
780 gGrid->Cd(workdir);
781 }
782 // Exit if any error up to now
783 if (error) return kFALSE;
784 // Set JDL fields
785 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 786 fGridJDL->SetExecutable(fExecutable);
c57f56b7 787// fGridJDL->SetTTL((UInt_t)fTTL);
788 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
789 if (fMaxInitFailed > 0)
790 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
791 if (fSplitMaxInputFileNumber > 0)
792 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
793 if (fSplitMode.Length())
794 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
795// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
796 if (fAliROOTVersion.Length())
797 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
798 if (fROOTVersion.Length())
799 fGridJDL->AddToPackages("ROOT", fROOTVersion);
800 if (fAPIVersion.Length())
801 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
648174cf 802 if (!fExternalPackages.IsNull()) {
803 arr = fExternalPackages.Tokenize(" ");
804 TIter next(arr);
805 while ((os=(TObjString*)next())) {
806 TString pkgname = os->GetString();
807 Int_t index = pkgname.Index("::");
808 TString pkgversion = pkgname(index+2, pkgname.Length());
809 pkgname.Remove(index);
810 fGridJDL->AddToPackages(pkgname, pkgversion);
811 }
812 delete arr;
813 }
c57f56b7 814 fGridJDL->SetInputDataListFormat(fInputFormat);
815 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 816 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 817 TString analysisFile = fExecutable;
818 analysisFile.ReplaceAll(".sh", ".root");
819 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 820 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
821 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
822 if (fAdditionalLibs.Length()) {
823 arr = fAdditionalLibs.Tokenize(" ");
824 TIter next(arr);
825 while ((os=(TObjString*)next())) {
826 if (os->GetString().Contains(".so")) continue;
827 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
828 }
829 delete arr;
830 }
4e5c5506 831 if (fPackages) {
832 TIter next(fPackages);
833 TObject *obj;
834 while ((obj=next()))
835 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
836 }
c57f56b7 837 if (fOutputArchive.Length()) {
838 arr = fOutputArchive.Tokenize(" ");
839 TIter next(arr);
840 while ((os=(TObjString*)next()))
43da816a 841 if (!os->GetString().Contains("@") && fCloseSE.Length())
842 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
843 else
844 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 845 delete arr;
846 }
c57f56b7 847 arr = fOutputFiles.Tokenize(" ");
848 TIter next(arr);
43da816a 849 while ((os=(TObjString*)next())) {
e1eaf596 850 // Ignore ouputs in jdl that are also in outputarchive
851 TString sout = os->GetString();
852 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
853 if (fOutputArchive.Contains(sout)) continue;
43da816a 854 if (!os->GetString().Contains("@") && fCloseSE.Length())
855 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
856 else
857 fGridJDL->AddToOutputSandbox(os->GetString());
858 }
c57f56b7 859 delete arr;
860// fGridJDL->SetPrice((UInt_t)fPrice);
861 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 862 TString validationScript = fExecutable;
863 validationScript.ReplaceAll(".sh", "_validation.sh");
864 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 865 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 866 // Write a jdl with 2 input parameters: collection name and output dir name.
867 WriteJDL(copy);
c57f56b7 868 }
869 // Copy jdl to grid workspace
a8739e8a 870 if (copy) {
648174cf 871 if (TestBit(AliAnalysisGrid::kSubmit)) {
872 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
873 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
874 if (fProductionMode)
875 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
876 if (FileExists(locjdl)) gGrid->Rm(locjdl);
877 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
878 }
c57f56b7 879 if (fAdditionalLibs.Length()) {
e7c71df0 880 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 881 TObjString *os;
882 TIter next(arr);
883 while ((os=(TObjString*)next())) {
c57f56b7 884 if (os->GetString().Contains(".so")) continue;
4e5c5506 885 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 886 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
887 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
888 }
889 delete arr;
890 }
4e5c5506 891 if (fPackages) {
892 TIter next(fPackages);
893 TObject *obj;
894 while ((obj=next())) {
895 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
896 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
897 }
898 }
c57f56b7 899 }
900 return kTRUE;
901}
902
a8739e8a 903//______________________________________________________________________________
d2a409b2 904Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 905{
906// Writes one or more JDL's corresponding to findex. If findex is negative,
907// all run numbers are considered in one go (jdl). For non-negative indices
908// they correspond to the indices in the array fInputFiles.
909 if (!fInputFiles) return kFALSE;
910 TObjString *os;
a8739e8a 911 TString workdir = gGrid->GetHomeDirectory();
912 workdir += fGridWorkingDir;
d2a409b2 913
914 if (!fRunNumbers.Length() && !fRunRange[0]) {
915 // One jdl with no parameters in case input data is specified by name.
a8739e8a 916 TIter next(fInputFiles);
917 while ((os=(TObjString*)next()))
918 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
648174cf 919 if (!fOutputSingle.IsNull())
920 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
921 else
922 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 923 } else {
d2a409b2 924 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
925 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
648174cf 926 if (!fOutputSingle.IsNull())
927 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
e1eaf596 928 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 929 }
930
931
932 // Generate the JDL as a string
933 TString sjdl = fGridJDL->Generate();
934 Int_t index;
935 index = sjdl.Index("Executable");
936 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
937 index = sjdl.Index("Split ");
648174cf 938 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
a8739e8a 939 index = sjdl.Index("SplitMaxInputFileNumber");
940 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
941 index = sjdl.Index("InputDataCollection");
942 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 943 index = sjdl.Index("InputFile");
944 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
945 index = sjdl.Index("InputDataList ");
946 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
947 index = sjdl.Index("InputDataListFormat");
948 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
949 index = sjdl.Index("Price");
950 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
951 index = sjdl.Index("Requirements");
952 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
953 index = sjdl.Index("Packages");
954 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
955 index = sjdl.Index("User =");
956 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
957 index = sjdl.Index("TTL");
958 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
959 index = sjdl.Index("OutputFile");
960 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
961 index = sjdl.Index("OutputDir");
962 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
963 index = sjdl.Index("OutputArchive");
964 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
965 index = sjdl.Index("MaxInitFailed");
966 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
967 index = sjdl.Index("MasterResubmitThreshold");
968 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
969 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
970 index = sjdl.Index("Validationcommand");
971 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
972 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
973 sjdl.ReplaceAll("(member", "\n (member");
974 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
975 sjdl.ReplaceAll("{", "{\n ");
976 sjdl.ReplaceAll("};", "\n};");
977 sjdl.ReplaceAll("{\n \n", "{\n");
978 sjdl.ReplaceAll("\n\n", "\n");
979 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
980 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 981 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 982 index = sjdl.Index("JDLVariables");
983 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
984 // Write jdl to file
a8739e8a 985 ofstream out;
d2a409b2 986 out.open(fJDLName.Data(), ios::out);
a8739e8a 987 if (out.bad()) {
d2a409b2 988 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 989 return kFALSE;
990 }
991 out << sjdl << endl;
992
993 // Copy jdl to grid workspace
994 if (!copy) {
995 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
996 } else {
d2a409b2 997 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
998 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 999 if (fProductionMode)
b5fe9cba 1000 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 1001 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 1002 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 1003 }
1004 return kTRUE;
1005}
1006
c57f56b7 1007//______________________________________________________________________________
5513444a 1008Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 1009{
1010// Returns true if file exists.
5513444a 1011 if (!gGrid) return kFALSE;
c57f56b7 1012 TGridResult *res = gGrid->Ls(lfn);
1013 if (!res) return kFALSE;
1014 TMap *map = dynamic_cast<TMap*>(res->At(0));
1015 if (!map) {
1016 delete res;
1017 return kFALSE;
1018 }
1019 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1020 if (!objs || !objs->GetString().Length()) {
1021 delete res;
1022 return kFALSE;
1023 }
1024 delete res;
1025 return kTRUE;
1026}
1027
923e2ca5 1028//______________________________________________________________________________
1029Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1030{
1031// Returns true if directory exists. Can be also a path.
1032 if (!gGrid) return kFALSE;
1033 // Check if dirname is a path
1034 TString dirstripped = dirname;
1035 dirstripped = dirstripped.Strip();
1036 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1037 TString dir = gSystem->BaseName(dirstripped);
1038 dir += "/";
1039 TString path = gSystem->DirName(dirstripped);
1040 TGridResult *res = gGrid->Ls(path, "-F");
1041 if (!res) return kFALSE;
1042 TIter next(res);
1043 TMap *map;
1044 TObject *obj;
1045 while ((map=dynamic_cast<TMap*>(next()))) {
1046 obj = map->GetValue("name");
1047 if (!obj) break;
1048 if (dir == obj->GetName()) {
1049 delete res;
1050 return kTRUE;
1051 }
1052 }
1053 delete res;
1054 return kFALSE;
1055}
1056
c57f56b7 1057//______________________________________________________________________________
1058void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1059{
1060// Check input data type.
1061 is_collection = kFALSE;
1062 is_xml = kFALSE;
1063 use_tags = kFALSE;
1064 if (!gGrid) {
1065 Error("CheckDataType", "No connection to grid");
1066 return;
1067 }
1068 is_collection = IsCollection(lfn);
1069 TString msg = "\n##### file: ";
1070 msg += lfn;
1071 if (is_collection) {
1072 msg += " type: raw_collection;";
1073 // special treatment for collections
1074 is_xml = kFALSE;
1075 // check for tag files in the collection
1076 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1077 if (!res) {
1078 msg += " using_tags: No (unknown)";
1079 Info("CheckDataType", msg.Data());
1080 return;
1081 }
1082 const char* typeStr = res->GetKey(0, "origLFN");
1083 if (!typeStr || !strlen(typeStr)) {
1084 msg += " using_tags: No (unknown)";
1085 Info("CheckDataType", msg.Data());
1086 return;
1087 }
1088 TString file = typeStr;
1089 use_tags = file.Contains(".tag");
1090 if (use_tags) msg += " using_tags: Yes";
1091 else msg += " using_tags: No";
1092 Info("CheckDataType", msg.Data());
1093 return;
1094 }
1095 TString slfn(lfn);
1096 slfn.ToLower();
1097 is_xml = slfn.Contains(".xml");
1098 if (is_xml) {
1099 // Open xml collection and check if there are tag files inside
1100 msg += " type: xml_collection;";
1101 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1102 if (!coll) {
1103 msg += " using_tags: No (unknown)";
1104 Info("CheckDataType", msg.Data());
1105 return;
1106 }
1107 TMap *map = coll->Next();
1108 if (!map) {
1109 msg += " using_tags: No (unknown)";
1110 Info("CheckDataType", msg.Data());
1111 return;
1112 }
1113 map = (TMap*)map->GetValue("");
1114 TString file;
1115 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1116 use_tags = file.Contains(".tag");
1117 delete coll;
1118 if (use_tags) msg += " using_tags: Yes";
1119 else msg += " using_tags: No";
1120 Info("CheckDataType", msg.Data());
1121 return;
1122 }
1123 use_tags = slfn.Contains(".tag");
1124 if (slfn.Contains(".root")) msg += " type: root file;";
1125 else msg += " type: unhnown file;";
1126 if (use_tags) msg += " using_tags: Yes";
1127 else msg += " using_tags: No";
1128 Info("CheckDataType", msg.Data());
1129}
1130
4e5c5506 1131//______________________________________________________________________________
1132void AliAnalysisAlien::EnablePackage(const char *package)
1133{
1134// Enables a par file supposed to exist in the current directory.
1135 TString pkg(package);
1136 pkg.ReplaceAll(".par", "");
1137 pkg += ".par";
1138 if (gSystem->AccessPathName(pkg)) {
1139 Error("EnablePackage", "Package %s not found", pkg.Data());
1140 return;
1141 }
1142 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1143 Info("EnablePackage", "AliEn plugin will use .par packages");
1144 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1145 if (!fPackages) {
1146 fPackages = new TObjArray();
1147 fPackages->SetOwner();
1148 }
1149 fPackages->Add(new TObjString(pkg));
1150}
1151
d2a409b2 1152//______________________________________________________________________________
1153const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1154{
1155// Get job status for all jobs with jobid>jobidstart.
1156 static char mstatus[20];
1157 mstatus[0] = '\0';
1158 nrunning = 0;
1159 nwaiting = 0;
1160 nerror = 0;
1161 ndone = 0;
1162 TGridJobStatusList *list = gGrid->Ps("");
1163 if (!list) return mstatus;
1164 Int_t nentries = list->GetSize();
1165 TGridJobStatus *status;
1166 Int_t pid;
1167 for (Int_t ijob=0; ijob<nentries; ijob++) {
1168 status = (TGridJobStatus *)list->At(ijob);
1169 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1170 if (pid<jobidstart) continue;
1171 if (pid == lastid) {
1172 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1173 }
1174 switch (status->GetStatus()) {
1175 case TGridJobStatus::kWAITING:
1176 nwaiting++; break;
1177 case TGridJobStatus::kRUNNING:
1178 nrunning++; break;
1179 case TGridJobStatus::kABORTED:
1180 case TGridJobStatus::kFAIL:
1181 case TGridJobStatus::kUNKNOWN:
1182 nerror++; break;
1183 case TGridJobStatus::kDONE:
1184 ndone++;
1185 }
1186 }
1187 list->Delete();
1188 delete list;
1189 return mstatus;
1190}
1191
c57f56b7 1192//______________________________________________________________________________
1193Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1194{
1195// Returns true if file is a collection. Functionality duplicated from
1196// TAlien::Type() because we don't want to directly depend on TAlien.
1197 if (!gGrid) {
1198 Error("IsCollection", "No connection to grid");
1199 return kFALSE;
1200 }
1201 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1202 if (!res) return kFALSE;
1203 const char* typeStr = res->GetKey(0, "type");
1204 if (!typeStr || !strlen(typeStr)) return kFALSE;
1205 if (!strcmp(typeStr, "collection")) return kTRUE;
1206 delete res;
1207 return kFALSE;
1208}
1209
16a4353c 1210//______________________________________________________________________________
1211void AliAnalysisAlien::Print(Option_t *) const
1212{
1213// Print current plugin settings.
84fcd93f 1214 printf("### AliEn analysis plugin current settings ###\n");
1215 printf("= Production mode:______________________________ %d\n", fProductionMode);
1216 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1217 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1218 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1219 if (fUser.Length())
84fcd93f 1220 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1221 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1222 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1223 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1224 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1225 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1226 if (fRunNumbers.Length())
84fcd93f 1227 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1228 if (fRunRange[0])
5fce53f4 1229 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
16a4353c 1230 if (!fRunRange[0] && !fRunNumbers.Length()) {
1231 TIter next(fInputFiles);
1232 TObject *obj;
1233 TString list;
1234 while ((obj=next())) list += obj->GetName();
84fcd93f 1235 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1236 }
1237 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1238 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1239 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1240 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1241 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1242 printf("=====================================================================\n");
1243 printf("= Job price: ___________________________________ %d\n", fPrice);
1244 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1245 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1246 if (fMaxInitFailed>0)
84fcd93f 1247 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1248 if (fMasterResubmitThreshold>0)
84fcd93f 1249 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1250 if (fNrunsPerMaster>0)
84fcd93f 1251 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1252 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1253 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
16a4353c 1254 if (fArguments.Length())
84fcd93f 1255 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1256 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1257 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1258 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1259 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1260 if (fDatasetName)
84fcd93f 1261 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1262 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1263 if (fIncludePath.Data())
84fcd93f 1264 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1265 if (fCloseSE.Length())
84fcd93f 1266 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1267 if (fFriendChainName.Length())
84fcd93f 1268 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1269 if (fPackages) {
1270 TIter next(fPackages);
1271 TObject *obj;
1272 TString list;
1273 while ((obj=next())) list += obj->GetName();
84fcd93f 1274 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1275 }
1276}
1277
c57f56b7 1278//______________________________________________________________________________
1279void AliAnalysisAlien::SetDefaults()
1280{
1281// Set default values for everything. What cannot be filled will be left empty.
1282 if (fGridJDL) delete fGridJDL;
1283 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1284 fPrice = 1;
1285 fTTL = 30000;
1286 fSplitMaxInputFileNumber = 100;
1287 fMaxInitFailed = 0;
1288 fMasterResubmitThreshold = 0;
bb885a9e 1289 fNtestFiles = 10;
a8739e8a 1290 fRunRange[0] = 0;
1291 fRunRange[1] = 0;
319593fb 1292 fNrunsPerMaster = 1;
16a4353c 1293 fMaxMergeFiles = 100;
c57f56b7 1294 fRunNumbers = "";
1295 fExecutable = "analysis.sh";
0a1c1f7f 1296 fExecutableCommand = "root -b -q";
c57f56b7 1297 fArguments = "";
1298 fAnalysisMacro = "myAnalysis.C";
1299 fAnalysisSource = "";
1300 fAdditionalLibs = "";
1301 fSplitMode = "se";
1302 fAPIVersion = "";
1303 fROOTVersion = "";
1304 fAliROOTVersion = "";
1305 fUser = ""; // Your alien user name
1306 fGridWorkingDir = "";
1307 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1308 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1309 fFriendChainName = "";
c57f56b7 1310 fGridOutputDir = "output";
1311 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1312 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1313 fInputFormat = "xml-single";
1314 fJDLName = "analysis.jdl";
c6cb3634 1315 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1316 fMergeExcludes = "";
c57f56b7 1317}
1318
1319//______________________________________________________________________________
1320Bool_t AliAnalysisAlien::MergeOutputs()
1321{
1322// Merge analysis outputs existing in the AliEn space.
1323 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1324 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1325 if (!Connect()) {
1326 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1327 return kFALSE;
1328 }
1329 // Get the output path
d2a409b2 1330 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1331 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1332 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1333 return kFALSE;
1334 }
1335 if (!fOutputFiles.Length()) {
1336 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1337 return kFALSE;
1338 }
1339 TObjArray *list = fOutputFiles.Tokenize(" ");
1340 TIter next(list);
1341 TObjString *str;
1342 TString command;
1343 TString output_file;
16a4353c 1344 TString output_chunk;
1345 TString previous_chunk;
1346 Int_t count_chunk = 0;
1347 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1348 Bool_t merged = kTRUE;
1349 while((str=(TObjString*)next())) {
1350 output_file = str->GetString();
1351 Int_t index = output_file.Index("@");
1352 if (index > 0) output_file.Remove(index);
319593fb 1353 // Skip already merged outputs
1354 if (!gSystem->AccessPathName(output_file)) {
1355 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1356 continue;
1357 }
bb885a9e 1358 if (fMergeExcludes.Length() &&
1359 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1360 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1361 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1362 printf("command: %s\n", command.Data());
c57f56b7 1363 TGridResult *res = gGrid->Command(command);
1364 if (!res) continue;
1365 TFileMerger *fm = 0;
1366 TIter nextmap(res);
ab254fd1 1367 TMap *map = 0;
16a4353c 1368 previous_chunk = "";
1369 count_chunk = 0;
319593fb 1370 // Check if there is a merge operation to resume
1371 output_chunk = output_file;
1372 output_chunk.ReplaceAll(".root", "_*.root");
1373 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1374 while (1) {
1375 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1376 if (!map) {
1377 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1378 delete res;
1379 return kFALSE;
1380 }
1381 output_chunk = output_file;
1382 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1383 printf("%s\n", output_chunk.Data());
319593fb 1384 count_chunk++;
1385 if (gSystem->AccessPathName(output_chunk)) continue;
1386 // Merged file with chunks up to <count_chunk> found
84fcd93f 1387 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1388 previous_chunk = output_chunk;
1389 break;
1390 }
1391 }
16a4353c 1392 count_zero = fMaxMergeFiles;
c57f56b7 1393 while ((map=(TMap*)nextmap())) {
16a4353c 1394 // Loop 'find' results and get next LFN
1395 if (count_zero == fMaxMergeFiles) {
1396 // First file in chunk - create file merger and add previous chunk if any.
1397 fm = new TFileMerger(kFALSE);
1398 fm->SetFastMethod(kTRUE);
1399 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1400 output_chunk = output_file;
1401 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1402 }
1403 // If last file found, put merged results in the output file
1404 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1405 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1406 if (!objs || !objs->GetString().Length()) {
16a4353c 1407 // Nothing found - skip this output
c57f56b7 1408 delete res;
16a4353c 1409 delete fm;
1410 break;
1411 }
1412 // Add file to be merged and decrement chunk counter.
1413 fm->AddFile(objs->GetString());
1414 count_zero--;
1415 if (count_zero==0 || map == res->Last()) {
1416 fm->OutputFile(output_chunk);
1417 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1418 // Nothing found - skip this output
1419 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1420 delete res;
1421 delete fm;
1422 break;
1423 }
1424 // Merge the outputs, then go to next chunk
1425 if (!fm->Merge()) {
1426 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1427 delete res;
1428 delete fm;
1429 merged = kFALSE;
1430 break;
1431 } else {
1432 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1433 gSystem->Unlink(previous_chunk);
16a4353c 1434 }
1435 if (map == res->Last()) {
1436 delete res;
1437 delete fm;
1438 break;
1439 }
1440 count_chunk++;
1441 count_zero = fMaxMergeFiles;
1442 previous_chunk = output_chunk;
c57f56b7 1443 }
c57f56b7 1444 }
c57f56b7 1445 }
1446 if (!merged) {
1447 Error("MergeOutputs", "Terminate() will NOT be executed");
1448 }
1449 return merged;
1450}
1451
bb885a9e 1452//______________________________________________________________________________
1453void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1454{
1455// Use the output files connected to output containers from the analysis manager
1456// rather than the files defined by SetOutputFiles
1457 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1458 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1459 analysis manager");
1460 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1461}
1462
5513444a 1463//______________________________________________________________________________
1464Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1465{
1466// Start remote grid analysis.
1467
43da816a 1468 // Check if output files have to be taken from the analysis manager
1469 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1470 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1471 if (!mgr || !mgr->IsInitialized()) {
1472 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1473 return kFALSE;
43da816a 1474 }
1475 fOutputFiles = "";
1476 TIter next(mgr->GetOutputs());
1477 AliAnalysisDataContainer *output;
1478 while ((output=(AliAnalysisDataContainer*)next())) {
1479 const char *filename = output->GetFileName();
1480 if (!(strcmp(filename, "default"))) {
1481 if (!mgr->GetOutputEventHandler()) continue;
1482 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1483 }
84fcd93f 1484 if (fOutputFiles.Contains(filename)) continue;
43da816a 1485 if (fOutputFiles.Length()) fOutputFiles += " ";
1486 fOutputFiles += filename;
1487 }
c07b9ce2 1488 // Add extra files registered to the analysis manager
1489 if (mgr->GetExtraFiles().Length()) {
1490 if (fOutputFiles.Length()) fOutputFiles += " ";
1491 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1492 }
43da816a 1493 }
f7b1cbc2 1494// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1495 if (TestBit(AliAnalysisGrid::kOffline)) {
1496 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1497 \n there nor any job run. You can revise the JDL and analysis \
1498 \n macro then run the same in \"submit\" mode.");
1499 } else if (TestBit(AliAnalysisGrid::kTest)) {
1500 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1501 \n dataset.");
1502 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1503 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1504 \n space and job submitted.");
1505 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1506 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1507 return kTRUE;
c57f56b7 1508 } else {
1509 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1510 }
1511
1512 if (!Connect()) {
1513 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1514 return kFALSE;
16a4353c 1515 }
1516 Print();
c57f56b7 1517 if (!CheckInputData()) {
1518 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1519 return kFALSE;
c57f56b7 1520 }
1521 CreateDataset(fDataPattern);
1522 WriteAnalysisFile();
1523 WriteAnalysisMacro();
1524 WriteExecutable();
1525 WriteValidationScript();
5513444a 1526 if (!CreateJDL()) return kFALSE;
1527 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1528 if (TestBit(AliAnalysisGrid::kTest)) {
1529 // Locally testing the analysis
1530 Info("StartAnalysis", "\n_______________________________________________________________________ \
1531 \n Running analysis script in a daughter shell as on a worker node \
1532 \n_______________________________________________________________________");
1533 TObjArray *list = fOutputFiles.Tokenize(" ");
1534 TIter next(list);
1535 TObjString *str;
1536 TString output_file;
1537 while((str=(TObjString*)next())) {
1538 output_file = str->GetString();
1539 Int_t index = output_file.Index("@");
1540 if (index > 0) output_file.Remove(index);
43da816a 1541 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1542 }
1543 delete list;
1544 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1545 TString validationScript = fExecutable;
1546 validationScript.ReplaceAll(".sh", "_validation.sh");
1547 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1548// gSystem->Exec("cat stdout");
5513444a 1549 return kFALSE;
c57f56b7 1550 }
5513444a 1551 // Check if submitting is managed by LPM manager
a3e84053 1552 if (fProductionMode) {
5513444a 1553 TString prodfile = fJDLName;
1554 prodfile.ReplaceAll(".jdl", ".prod");
1555 WriteProductionFile(prodfile);
1556 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1557 return kFALSE;
1558 }
a8739e8a 1559 // Submit AliEn job(s)
d2a409b2 1560 gGrid->Cd(fGridOutputDir);
a8739e8a 1561 TGridResult *res;
c57f56b7 1562 TString jobID = "";
d2a409b2 1563 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1564 // Submit a given xml or a set of runs
a8739e8a 1565 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1566 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1567 if (res) {
1568 const char *cjobId = res->GetKey(0,"jobId");
1569 if (!cjobId) {
1570 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1571 return kFALSE;
a8739e8a 1572 } else {
1573 Info("StartAnalysis", "\n_______________________________________________________________________ \
1574 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1575 \n_______________________________________________________________________",
1576 fJDLName.Data(), cjobId);
1577 jobID = cjobId;
1578 }
1579 delete res;
1580 }
1581 } else {
d2a409b2 1582 // Submit for a range of enumeration of runs.
1583 Submit();
c57f56b7 1584 }
a8739e8a 1585
c57f56b7 1586 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1587 \n You may exit at any time and terminate the job later using the option <terminate> \
1588 \n ##################################################################################", jobID.Data());
bb885a9e 1589 gSystem->Exec("aliensh");
5513444a 1590 return kTRUE;
c57f56b7 1591}
1592
d2a409b2 1593//______________________________________________________________________________
1594void AliAnalysisAlien::Submit()
1595{
1596// Submit all master jobs.
1597 Int_t nmasterjobs = fInputFiles->GetEntries();
1598 Long_t tshoot = gSystem->Now();
1599 if (!fNsubmitted) SubmitNext();
1600 while (fNsubmitted < nmasterjobs) {
1601 Long_t now = gSystem->Now();
1602 if ((now-tshoot)>30000) {
1603 tshoot = now;
1604 SubmitNext();
1605 }
1606 }
1607}
1608
1609//______________________________________________________________________________
1610void AliAnalysisAlien::SubmitNext()
1611{
1612// Submit next bunch of master jobs if the queue is free.
1613 static Bool_t iscalled = kFALSE;
1614 static Int_t firstmaster = 0;
1615 static Int_t lastmaster = 0;
1616 static Int_t npermaster = 0;
1617 if (iscalled) return;
1618 iscalled = kTRUE;
1619 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1620 Int_t ntosubmit = 0;
1621 TGridResult *res;
1622 TString jobID = "";
1623 if (!fNsubmitted) ntosubmit = 1;
1624 else {
1625 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1626 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1627 // If last master not split, just return
1628 if (status != "SPLIT") {iscalled = kFALSE; return;}
1629 // No more than 100 waiting jobs
1630 if (nwaiting>100) {iscalled = kFALSE; return;}
1631 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1632 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1633 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1634 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1635 }
1636 Int_t nmasterjobs = fInputFiles->GetEntries();
1637 for (Int_t i=0; i<ntosubmit; i++) {
1638 // Submit for a range of enumeration of runs.
1639 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1640 TString query;
1641 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1642 printf("********* %s\n",query.Data());
d2a409b2 1643 res = gGrid->Command(query);
1644 if (res) {
98ca124f 1645 TString cjobId1 = res->GetKey(0,"jobId");
1646 if (!cjobId1.Length()) {
d2a409b2 1647 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1648 iscalled = kFALSE;
1649 return;
1650 } else {
1651 Info("StartAnalysis", "\n_______________________________________________________________________ \
1652 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1653 \n_______________________________________________________________________",
98ca124f 1654 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1655 jobID += cjobId1;
1656 jobID += " ";
98ca124f 1657 lastmaster = cjobId1.Atoi();
d2a409b2 1658 if (!firstmaster) firstmaster = lastmaster;
1659 fNsubmitted++;
1660 }
1661 delete res;
1662 }
1663 }
1664 iscalled = kFALSE;
1665}
1666
c57f56b7 1667//______________________________________________________________________________
1668void AliAnalysisAlien::WriteAnalysisFile()
1669{
f10e8481 1670// Write current analysis manager into the file <analysisFile>
1671 TString analysisFile = fExecutable;
1672 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1673 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1674 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1675 if (!mgr || !mgr->IsInitialized()) {
1676 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1677 return;
1678 }
1679 // Check analysis type
1680 TObject *handler;
1681 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1682 handler = (TObject*)mgr->GetInputEventHandler();
1683 if (handler) {
1684 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1685 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1686 }
1687 TDirectory *cdir = gDirectory;
f10e8481 1688 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1689 if (file) {
1690 mgr->Write();
1691 delete file;
1692 }
1693 if (cdir) cdir->cd();
f10e8481 1694 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1695 }
1696 Bool_t copy = kTRUE;
1697 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1698 if (copy) {
1699 CdWork();
1700 TString workdir = gGrid->GetHomeDirectory();
1701 workdir += fGridWorkingDir;
f10e8481 1702 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1703 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1704 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1705 }
1706}
1707
1708//______________________________________________________________________________
1709void AliAnalysisAlien::WriteAnalysisMacro()
1710{
1711// Write the analysis macro that will steer the analysis in grid mode.
1712 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1713 ofstream out;
1714 out.open(fAnalysisMacro.Data(), ios::out);
1715 if (!out.good()) {
1716 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1717 return;
1718 }
1719 TString func = fAnalysisMacro;
1720 TString type = "ESD";
1721 TString comment = "// Analysis using ";
1722 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1723 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1724 type = "AOD";
1725 comment += "AOD";
1726 }
0df6ccf2 1727 if (type!="AOD" && fFriendChainName!="") {
1728 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1729 return;
1730 }
c57f56b7 1731 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1732 else comment += " data";
1733 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1734 func.ReplaceAll(".C", "");
1735 out << "void " << func.Data() << "()" << endl;
1736 out << "{" << endl;
1737 out << comment.Data() << endl;
1738 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1739 out << " TStopwatch timer;" << endl;
1740 out << " timer.Start();" << endl << endl;
c57f56b7 1741 out << "// load base root libraries" << endl;
1742 out << " gSystem->Load(\"libTree\");" << endl;
1743 out << " gSystem->Load(\"libGeom\");" << endl;
1744 out << " gSystem->Load(\"libVMC\");" << endl;
1745 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 1746 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
57377eb5 1747 out << "// Load analysis framework libraries" << endl;
4e5c5506 1748 if (!fPackages) {
4e5c5506 1749 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1750 out << " gSystem->Load(\"libESD\");" << endl;
1751 out << " gSystem->Load(\"libAOD\");" << endl;
1752 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1753 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1754 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1755 } else {
4e5c5506 1756 TIter next(fPackages);
1757 TObject *obj;
57377eb5 1758 TString pkgname;
1759 Bool_t hasSTEERBase = kFALSE;
1760 Bool_t hasESD = kFALSE;
1761 Bool_t hasAOD = kFALSE;
1762 Bool_t hasANALYSIS = kFALSE;
1763 Bool_t hasANALYSISalice = kFALSE;
1764 Bool_t hasCORRFW = kFALSE;
1765 while ((obj=next())) {
1766 pkgname = obj->GetName();
4478e6f1 1767 if (pkgname == "STEERBase" ||
1768 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1769 if (pkgname == "ESD" ||
1770 pkgname == "ESD.par") hasESD = kTRUE;
1771 if (pkgname == "AOD" ||
1772 pkgname == "AOD.par") hasAOD = kTRUE;
1773 if (pkgname == "ANALYSIS" ||
1774 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1775 if (pkgname == "ANALYSISalice" ||
1776 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1777 if (pkgname == "CORRFW" ||
1778 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1779 }
1780 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1781 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1782 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1783 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1784 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1785 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1786 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1787 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1788 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1789 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1790 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1791 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1792 out << "// Compile other par packages" << endl;
1793 next.Reset();
fcc9bb6f 1794 while ((obj=next())) {
1795 pkgname = obj->GetName();
4478e6f1 1796 if (pkgname == "STEERBase" ||
1797 pkgname == "STEERBase.par" ||
1798 pkgname == "ESD" ||
1799 pkgname == "ESD.par" ||
1800 pkgname == "AOD" ||
1801 pkgname == "AOD.par" ||
1802 pkgname == "ANALYSIS" ||
1803 pkgname == "ANALYSIS.par" ||
1804 pkgname == "ANALYSISalice" ||
1805 pkgname == "ANALYSISalice.par" ||
1806 pkgname == "CORRFW" ||
1807 pkgname == "CORRFW.par") continue;
4e5c5506 1808 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1809 }
4e5c5506 1810 }
43da816a 1811 out << "// include path" << endl;
1812 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1813 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1814 if (fAdditionalLibs.Length()) {
1815 out << "// Add aditional AliRoot libraries" << endl;
1816 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1817 TIter next(list);
1818 TObjString *str;
1819 while((str=(TObjString*)next())) {
1820 if (str->GetString().Contains(".so"))
1821 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1822 }
1823 if (list) delete list;
1824 }
1825 out << endl;
c57f56b7 1826 out << "// analysis source to be compiled at runtime (if any)" << endl;
1827 if (fAnalysisSource.Length()) {
1828 TObjArray *list = fAnalysisSource.Tokenize(" ");
1829 TIter next(list);
1830 TObjString *str;
1831 while((str=(TObjString*)next())) {
1832 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1833 }
1834 if (list) delete list;
1835 }
1836 out << endl;
1837 out << "// connect to AliEn and make the chain" << endl;
1838 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1839 if (IsUsingTags()) {
1840 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1841 } else {
76535c98 1842 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1843 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1844 } else {
1845 out << " // Check if the macro to create the chain was provided" << endl;
1846 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1847 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1848 out << " return;" << endl;
1849 out << " }" << endl;
1850 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1851 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1852 }
c57f56b7 1853 }
1854 out << "// read the analysis manager from file" << endl;
f10e8481 1855 TString analysisFile = fExecutable;
1856 analysisFile.ReplaceAll(".sh", ".root");
1857 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1858 out << " if (!file) return;" << endl;
1859 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1860 out << " AliAnalysisManager *mgr = 0;" << endl;
1861 out << " TKey *key;" << endl;
1862 out << " while ((key=(TKey*)nextkey())) {" << endl;
1863 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1864 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1865 out << " };" << endl;
1866 out << " if (!mgr) {" << endl;
f10e8481 1867 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1868 out << " return;" << endl;
1869 out << " }" << endl << endl;
1870 out << " mgr->PrintStatus();" << endl;
1871 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1872 out << " timer.Stop();" << endl;
1873 out << " timer.Print();" << endl;
c57f56b7 1874 out << "}" << endl << endl;
1875 if (IsUsingTags()) {
1876 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1877 out << "{" << endl;
1878 out << "// Create a chain using tags from the xml file." << endl;
1879 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1880 out << " if (!coll) {" << endl;
1881 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1882 out << " return NULL;" << endl;
1883 out << " }" << endl;
1884 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1885 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1886 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1887 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1888 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1889 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1890 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1891 out << " // Check if the cuts configuration file was provided" << endl;
1892 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1893 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1894 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1895 out << " }" << endl;
0df6ccf2 1896 if (fFriendChainName=="") {
1897 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1898 } else {
1899 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1900 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1901 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1902 }
c57f56b7 1903 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1904 out << " chain->ls();" << endl;
1905 out << " return chain;" << endl;
fcc9bb6f 1906 out << "}" << endl << endl;
c57f56b7 1907 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1908 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1909 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1910 msg += " AliLHCTagCuts *lhcCuts,\n";
1911 msg += " AliDetectorTagCuts *detCuts,\n";
1912 msg += " AliEventTagCuts *evCuts)";
1913 Info("WriteAnalysisMacro", msg.Data());
1914 }
0df6ccf2 1915 }
1916 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1917 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1918 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1919 out << "{" << endl;
1920 out << "// Create a chain using url's from xml file" << endl;
1921 out << " TString treename = type;" << endl;
1922 out << " treename.ToLower();" << endl;
1923 out << " treename += \"Tree\";" << endl;
e02fee64 1924 out << " printf(\"***************************************\\n\");" << endl;
1925 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1926 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1927 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1928 out << " if (!coll) {" << endl;
1929 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1930 out << " return NULL;" << endl;
1931 out << " }" << endl;
1932 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1933 if(fFriendChainName!="") {
1934 out << " TChain *chainFriend = new TChain(treename);" << endl;
1935 }
c57f56b7 1936 out << " coll->Reset();" << endl;
0df6ccf2 1937 out << " while (coll->Next()) {" << endl;
1938 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1939 if(fFriendChainName!="") {
1940 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1941 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1942 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1943 out << " chainFriend->Add(fileFriend.Data());" << endl;
1944 }
1945 out << " }" << endl;
c57f56b7 1946 out << " if (!chain->GetNtrees()) {" << endl;
1947 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1948 out << " return NULL;" << endl;
1949 out << " }" << endl;
0df6ccf2 1950 if(fFriendChainName!="") {
1951 out << " chain->AddFriend(chainFriend);" << endl;
1952 }
c57f56b7 1953 out << " return chain;" << endl;
fcc9bb6f 1954 out << "}" << endl << endl;
c57f56b7 1955 }
4e5c5506 1956 if (fPackages) {
fcc9bb6f 1957 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1958 out << "Bool_t SetupPar(const char *package) {" << endl;
1959 out << "// Compile the package and set it up." << endl;
1960 out << " TString pkgdir = package;" << endl;
1961 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1962 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1963 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1964 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1965 out << " // Check for BUILD.sh and execute" << endl;
1966 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1967 out << " printf(\"*******************************\\n\");" << endl;
1968 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1969 out << " printf(\"*******************************\\n\");" << endl;
1970 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1971 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1972 out << " gSystem->ChangeDirectory(cdir);" << endl;
1973 out << " return kFALSE;" << endl;
1974 out << " }" << endl;
1975 out << " } else {" << endl;
fcc9bb6f 1976 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1977 out << " gSystem->ChangeDirectory(cdir);" << endl;
1978 out << " return kFALSE;" << endl;
1979 out << " }" << endl;
1980 out << " // Check for SETUP.C and execute" << endl;
1981 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1982 out << " printf(\"*******************************\\n\");" << endl;
1983 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1984 out << " printf(\"*******************************\\n\");" << endl;
1985 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1986 out << " } else {" << endl;
fcc9bb6f 1987 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1988 out << " gSystem->ChangeDirectory(cdir);" << endl;
1989 out << " return kFALSE;" << endl;
1990 out << " }" << endl;
1991 out << " // Restore original workdir" << endl;
1992 out << " gSystem->ChangeDirectory(cdir);" << endl;
1993 out << " return kTRUE;" << endl;
1994 out << "}" << endl;
1995 }
c57f56b7 1996 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1997 }
1998 Bool_t copy = kTRUE;
1999 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2000 if (copy) {
2001 CdWork();
2002 TString workdir = gGrid->GetHomeDirectory();
2003 workdir += fGridWorkingDir;
2004 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2005 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2006 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2007 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2008 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2009 }
2010 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2011 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2012 }
2013}
2014
2015//______________________________________________________________________________
2016void AliAnalysisAlien::WriteExecutable()
2017{
2018// Generate the alien executable script.
2019 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2020 ofstream out;
2021 out.open(fExecutable.Data(), ios::out);
2022 if (out.bad()) {
5513444a 2023 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2024 return;
2025 }
2026 out << "#!/bin/bash" << endl;
c57f56b7 2027 out << "echo \"=========================================\"" << endl;
2028 out << "echo \"############## PATH : ##############\"" << endl;
2029 out << "echo $PATH" << endl;
2030 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2031 out << "echo $LD_LIBRARY_PATH" << endl;
2032 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2033 out << "echo $ROOTSYS" << endl;
2034 out << "echo \"############## which root : ##############\"" << endl;
2035 out << "which root" << endl;
2036 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2037 out << "echo $ALICE_ROOT" << endl;
2038 out << "echo \"############## which aliroot : ##############\"" << endl;
2039 out << "which aliroot" << endl;
2040 out << "echo \"=========================================\"" << endl << endl;
74a53467 2041 // Make sure we can properly compile par files
2042 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2043 out << fExecutableCommand << " ";
c57f56b7 2044 out << fAnalysisMacro.Data() << endl << endl;
2045 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
2046 }
2047 Bool_t copy = kTRUE;
2048 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2049 if (copy) {
2050 CdWork();
2051 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2052 TString bindir = Form("%s/bin", workdir.Data());
2053 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2054 workdir += fGridWorkingDir;
2055 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2056 if (FileExists(executable)) gGrid->Rm(executable);
2057 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2058 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2059 }
2060}
2061
2062//______________________________________________________________________________
5513444a 2063void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2064{
2065// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2066// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2067// Next lines: full_path_to_dataset XXX (XXX is a string)
2068// To submit, one has to: submit jdl XXX for all lines
2069 ofstream out;
2070 out.open(filename, ios::out);
2071 if (out.bad()) {
2072 Error("WriteProductionFile", "Bad file name: %s", filename);
2073 return;
2074 }
2075 TString workdir = gGrid->GetHomeDirectory();
2076 workdir += fGridWorkingDir;
f5e8c702 2077 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2078 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2079 out << locjdl << " " << njobspermaster << endl;
5513444a 2080 Int_t nmasterjobs = fInputFiles->GetEntries();
2081 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2082 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2083 }
2084 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2085 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2086}
2087
2088//______________________________________________________________________________
c57f56b7 2089void AliAnalysisAlien::WriteValidationScript()
2090{
2091// Generate the alien validation script.
2092 // Generate the validation script
2093 TObjString *os;
0d5d317c 2094 TString validationScript = fExecutable;
2095 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2096 if (!Connect()) {
2097 Error("WriteValidationScript", "Alien connection required");
2098 return;
2099 }
2100 TString out_stream = "";
2101 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2102 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2103 ofstream out;
0d5d317c 2104 out.open(validationScript, ios::out);
c57f56b7 2105 out << "#!/bin/bash" << endl;
2106 out << "##################################################" << endl;
2107 out << "validateout=`dirname $0`" << endl;
2108 out << "validatetime=`date`" << endl;
2109 out << "validated=\"0\";" << endl;
2110 out << "error=0" << endl;
2111 out << "if [ -z $validateout ]" << endl;
2112 out << "then" << endl;
2113 out << " validateout=\".\"" << endl;
2114 out << "fi" << endl << endl;
2115 out << "cd $validateout;" << endl;
2116 out << "validateworkdir=`pwd`;" << endl << endl;
2117 out << "echo \"*******************************************************\"" << out_stream << endl;
2118 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2119 out << "" << endl;
2120 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2121 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2122 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2123 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2124 out << "ls -la ./" << out_stream << endl;
2125 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2126 out << "##################################################" << endl;
ebec370a 2127
2128 out << "" << endl;
2129 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2130 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2131 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2132 out << "" << endl;
2133
2134 out << "if [ ! -f stderr ] ; then" << endl;
2135 out << " error=1" << endl;
2136 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2137 out << " echo \"Error = $error\" " << out_stream << endl;
2138 out << "fi" << endl;
2139
2140 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2141 out << " error=1" << endl;
2142 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2143 out << " echo \"$parArch\" " << out_stream << endl;
2144 out << " echo \"Error = $error\" " << out_stream << endl;
2145 out << "fi" << endl;
2146
2147 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2148 out << " error=1" << endl;
2149 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2150 out << " echo \"$segViol\" " << out_stream << endl;
2151 out << " echo \"Error = $error\" " << out_stream << endl;
2152 out << "fi" << endl;
2153
2154 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2155 out << " error=1" << endl;
2156 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2157 out << " echo \"$segFault\" " << out_stream << endl;
2158 out << " echo \"Error = $error\" " << out_stream << endl;
2159 out << "fi" << endl;
2160
2161 // Part dedicated to the specific analyses running into the train
2162
c57f56b7 2163 TObjArray *arr = fOutputFiles.Tokenize(" ");
2164 TIter next1(arr);
2165 TString output_file;
2166 while ((os=(TObjString*)next1())) {
2167 output_file = os->GetString();
2168 Int_t index = output_file.Index("@");
2169 if (index > 0) output_file.Remove(index);
2170 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2171 out << " error=1" << endl;
2172 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2173 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2174 out << "fi" << endl;
2175 }
2176 delete arr;
923e2ca5 2177 out << "if ! [ -f outputs_valid ] ; then" << endl;
2178 out << " error=1" << endl;
2179 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2180 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2181 out << "fi" << endl;
2182
c57f56b7 2183 out << "if [ $error = 0 ] ; then" << endl;
2184 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2185 out << "fi" << endl;
2186
2187 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2188 out << "echo \"*******************************************************\"" << out_stream << endl;
2189 out << "cd -" << endl;
2190 out << "exit $error" << endl;
2191 }
2192 Bool_t copy = kTRUE;
2193 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2194 if (copy) {
2195 CdWork();
2196 TString workdir = gGrid->GetHomeDirectory();
2197 workdir += fGridWorkingDir;
0d5d317c 2198 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2199 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2200 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2201 }
2202}