]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Set the running environment before the setup of the CDB in oder to provide
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
bb885a9e 35#include "AliVEventHandler.h"
36#include "AliAnalysisDataContainer.h"
c57f56b7 37#include "AliAnalysisAlien.h"
38
39ClassImp(AliAnalysisAlien)
40
41//______________________________________________________________________________
42AliAnalysisAlien::AliAnalysisAlien()
43 :AliAnalysisGrid(),
44 fGridJDL(NULL),
45 fPrice(0),
46 fTTL(0),
47 fSplitMaxInputFileNumber(0),
48 fMaxInitFailed(0),
49 fMasterResubmitThreshold(0),
bb885a9e 50 fNtestFiles(0),
319593fb 51 fNrunsPerMaster(0),
16a4353c 52 fMaxMergeFiles(0),
c57f56b7 53 fRunNumbers(),
54 fExecutable(),
55 fArguments(),
56 fAnalysisMacro(),
57 fAnalysisSource(),
58 fAdditionalLibs(),
59 fSplitMode(),
60 fAPIVersion(),
61 fROOTVersion(),
62 fAliROOTVersion(),
63 fUser(),
64 fGridWorkingDir(),
65 fGridDataDir(),
66 fDataPattern(),
67 fGridOutputDir(),
68 fOutputArchive(),
69 fOutputFiles(),
70 fInputFormat(),
e7c71df0 71 fDatasetName(),
c57f56b7 72 fJDLName(),
bb885a9e 73 fMergeExcludes(),
f965131e 74 fIncludePath(),
bb885a9e 75 fCloseSE(),
0df6ccf2 76 fFriendChainName(),
4e5c5506 77 fInputFiles(0),
78 fPackages(0)
c57f56b7 79{
80// Dummy ctor.
81 SetDefaults();
82}
83
84//______________________________________________________________________________
85AliAnalysisAlien::AliAnalysisAlien(const char *name)
86 :AliAnalysisGrid(name),
87 fGridJDL(NULL),
88 fPrice(0),
89 fTTL(0),
90 fSplitMaxInputFileNumber(0),
91 fMaxInitFailed(0),
92 fMasterResubmitThreshold(0),
bb885a9e 93 fNtestFiles(0),
319593fb 94 fNrunsPerMaster(0),
16a4353c 95 fMaxMergeFiles(0),
c57f56b7 96 fRunNumbers(),
97 fExecutable(),
98 fArguments(),
99 fAnalysisMacro(),
100 fAnalysisSource(),
101 fAdditionalLibs(),
102 fSplitMode(),
103 fAPIVersion(),
104 fROOTVersion(),
105 fAliROOTVersion(),
106 fUser(),
107 fGridWorkingDir(),
108 fGridDataDir(),
109 fDataPattern(),
110 fGridOutputDir(),
111 fOutputArchive(),
112 fOutputFiles(),
113 fInputFormat(),
e7c71df0 114 fDatasetName(),
c57f56b7 115 fJDLName(),
bb885a9e 116 fMergeExcludes(),
f965131e 117 fIncludePath(),
bb885a9e 118 fCloseSE(),
0df6ccf2 119 fFriendChainName(),
4e5c5506 120 fInputFiles(0),
121 fPackages(0)
c57f56b7 122{
123// Default ctor.
124 SetDefaults();
125}
126
127//______________________________________________________________________________
128AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
129 :AliAnalysisGrid(other),
130 fGridJDL(NULL),
131 fPrice(other.fPrice),
132 fTTL(other.fTTL),
133 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
134 fMaxInitFailed(other.fMaxInitFailed),
135 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 136 fNtestFiles(other.fNtestFiles),
319593fb 137 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 138 fMaxMergeFiles(other.fMaxMergeFiles),
c57f56b7 139 fRunNumbers(other.fRunNumbers),
140 fExecutable(other.fExecutable),
141 fArguments(other.fArguments),
142 fAnalysisMacro(other.fAnalysisMacro),
143 fAnalysisSource(other.fAnalysisSource),
144 fAdditionalLibs(other.fAdditionalLibs),
145 fSplitMode(other.fSplitMode),
146 fAPIVersion(other.fAPIVersion),
147 fROOTVersion(other.fROOTVersion),
148 fAliROOTVersion(other.fAliROOTVersion),
149 fUser(other.fUser),
150 fGridWorkingDir(other.fGridWorkingDir),
151 fGridDataDir(other.fGridDataDir),
152 fDataPattern(other.fDataPattern),
153 fGridOutputDir(other.fGridOutputDir),
154 fOutputArchive(other.fOutputArchive),
155 fOutputFiles(other.fOutputFiles),
156 fInputFormat(other.fInputFormat),
e7c71df0 157 fDatasetName(other.fDatasetName),
c57f56b7 158 fJDLName(other.fJDLName),
bb885a9e 159 fMergeExcludes(other.fMergeExcludes),
f965131e 160 fIncludePath(other.fIncludePath),
bb885a9e 161 fCloseSE(other.fCloseSE),
0df6ccf2 162 fFriendChainName(other.fFriendChainName),
4e5c5506 163 fInputFiles(0),
164 fPackages(0)
c57f56b7 165{
166// Copy ctor.
167 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 168 fRunRange[0] = other.fRunRange[0];
169 fRunRange[1] = other.fRunRange[1];
c57f56b7 170 if (other.fInputFiles) {
171 fInputFiles = new TObjArray();
172 TIter next(other.fInputFiles);
173 TObject *obj;
174 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
175 fInputFiles->SetOwner();
176 }
4e5c5506 177 if (other.fPackages) {
178 fPackages = new TObjArray();
179 TIter next(other.fPackages);
180 TObject *obj;
181 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
182 fPackages->SetOwner();
183 }
c57f56b7 184}
185
186//______________________________________________________________________________
187AliAnalysisAlien::~AliAnalysisAlien()
188{
189// Destructor.
190 if (fGridJDL) delete fGridJDL;
191 if (fInputFiles) delete fInputFiles;
4e5c5506 192 if (fPackages) delete fPackages;
c57f56b7 193}
194
195//______________________________________________________________________________
196AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
197{
198// Assignment.
199 if (this != &other) {
200 AliAnalysisGrid::operator=(other);
201 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
202 fPrice = other.fPrice;
203 fTTL = other.fTTL;
204 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
205 fMaxInitFailed = other.fMaxInitFailed;
206 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 207 fNtestFiles = other.fNtestFiles;
c57f56b7 208 fRunNumbers = other.fRunNumbers;
209 fExecutable = other.fExecutable;
210 fArguments = other.fArguments;
211 fAnalysisMacro = other.fAnalysisMacro;
212 fAnalysisSource = other.fAnalysisSource;
213 fAdditionalLibs = other.fAdditionalLibs;
214 fSplitMode = other.fSplitMode;
215 fAPIVersion = other.fAPIVersion;
216 fROOTVersion = other.fROOTVersion;
217 fAliROOTVersion = other.fAliROOTVersion;
218 fUser = other.fUser;
219 fGridWorkingDir = other.fGridWorkingDir;
220 fGridDataDir = other.fGridDataDir;
221 fDataPattern = other.fDataPattern;
222 fGridOutputDir = other.fGridOutputDir;
223 fOutputArchive = other.fOutputArchive;
224 fOutputFiles = other.fOutputFiles;
225 fInputFormat = other.fInputFormat;
e7c71df0 226 fDatasetName = other.fDatasetName;
c57f56b7 227 fJDLName = other.fJDLName;
bb885a9e 228 fMergeExcludes = other.fMergeExcludes;
f965131e 229 fIncludePath = other.fIncludePath;
bb885a9e 230 fCloseSE = other.fCloseSE;
0df6ccf2 231 fFriendChainName = other.fFriendChainName;
c57f56b7 232 if (other.fInputFiles) {
233 fInputFiles = new TObjArray();
234 TIter next(other.fInputFiles);
235 TObject *obj;
236 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
237 fInputFiles->SetOwner();
238 }
4e5c5506 239 if (other.fPackages) {
240 fPackages = new TObjArray();
241 TIter next(other.fPackages);
242 TObject *obj;
243 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
244 fPackages->SetOwner();
245 }
c57f56b7 246 }
247 return *this;
248}
249
f965131e 250//______________________________________________________________________________
251void AliAnalysisAlien::AddIncludePath(const char *path)
252{
253// Add include path in the remote analysis macro.
254 TString p(path);
255 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
256 else fIncludePath += Form("-I%s ", path);
257}
258
c57f56b7 259//______________________________________________________________________________
260void AliAnalysisAlien::AddRunNumber(Int_t run)
261{
262// Add a run number to the list of runs to be processed.
263 if (fRunNumbers.Length()) fRunNumbers += " ";
264 fRunNumbers += Form("%d", run);
265}
266
267//______________________________________________________________________________
268void AliAnalysisAlien::AddDataFile(const char *lfn)
269{
270// Adds a data file to the input to be analysed. The file should be a valid LFN
271// or point to an existing file in the alien workdir.
272 if (!fInputFiles) fInputFiles = new TObjArray();
273 fInputFiles->Add(new TObjString(lfn));
274}
275
276//______________________________________________________________________________
277Bool_t AliAnalysisAlien::Connect()
278{
279// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
280 if (gGrid && gGrid->IsConnected()) return kTRUE;
281 if (!gSystem->Getenv("alien_API_USER")) {
282 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
283 gSystem->Getenv("UID"));
284 return kFALSE;
285 }
286 if (!gGrid) {
287 Info("Connect", "Trying to connect to AliEn ...");
288 TGrid::Connect("alien://");
289 }
290 if (!gGrid || !gGrid->IsConnected()) {
291 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
292 return kFALSE;
293 }
294 fUser = gGrid->GetUser();
295 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
296 return kTRUE;
297}
298
299//______________________________________________________________________________
300void AliAnalysisAlien::CdWork()
301{
302// Check validity of alien workspace. Create directory if possible.
303 if (!Connect()) {
304 Error("CdWork", "Alien connection required");
305 return;
306 }
307 TString homedir = gGrid->GetHomeDirectory();
308 TString workdir = homedir + fGridWorkingDir;
309 if (!gGrid->Cd(workdir)) {
310 gGrid->Cd(homedir);
311 if (gGrid->Mkdir(workdir)) {
312 gGrid->Cd(fGridWorkingDir);
313 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
314 } else {
315 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
316 workdir.Data(), homedir.Data());
317 fGridWorkingDir = "";
318 }
319 }
320}
321
322//______________________________________________________________________________
323Bool_t AliAnalysisAlien::CheckInputData()
324{
325// Check validity of input data. If necessary, create xml files.
a8739e8a 326 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
327 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile()/SetRunRange().");
c57f56b7 328 return kFALSE;
329 }
330 // Process declared files
331 Bool_t is_collection = kFALSE;
332 Bool_t is_xml = kFALSE;
333 Bool_t use_tags = kFALSE;
334 Bool_t checked = kFALSE;
335 CdWork();
336 TString file;
337 TString workdir = gGrid->GetHomeDirectory();
338 workdir += fGridWorkingDir;
339 if (fInputFiles) {
340 TObjString *objstr;
341 TIter next(fInputFiles);
342 while ((objstr=(TObjString*)next())) {
343 file = workdir;
344 file += "/";
345 file += objstr->GetString();
346 // Store full lfn path
347 if (FileExists(file)) objstr->SetString(file);
348 else {
349 file = objstr->GetName();
350 if (!FileExists(objstr->GetName())) {
351 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
352 objstr->GetName(), workdir.Data());
353 return kFALSE;
354 }
355 }
356 Bool_t iscoll, isxml, usetags;
357 CheckDataType(file, iscoll, isxml, usetags);
358 if (!checked) {
359 checked = kTRUE;
360 is_collection = iscoll;
361 is_xml = isxml;
362 use_tags = usetags;
363 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
364 } else {
365 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
366 Error("CheckInputData", "Some conflict was found in the types of inputs");
367 return kFALSE;
368 }
369 }
370 }
371 }
372 // Process requested run numbers
a8739e8a 373 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 374 // Check validity of alien data directory
375 if (!fGridDataDir.Length()) {
376 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
377 return kFALSE;
378 }
379 if (!gGrid->Cd(fGridDataDir)) {
380 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
381 return kFALSE;
382 }
383 if (is_collection) {
384 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
385 return kFALSE;
386 }
387
388 if (checked && !is_xml) {
389 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
390 return kFALSE;
391 }
392 // Check validity of run number(s)
393 TObjArray *arr;
394 TObjString *os;
319593fb 395 Int_t nruns = 0;
396 TString schunk;
c57f56b7 397 TString path;
398 if (!checked) {
399 checked = kTRUE;
400 use_tags = fDataPattern.Contains("tag");
401 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
402 }
403 if (use_tags != fDataPattern.Contains("tag")) {
404 Error("CheckInputData", "Cannot mix input files using/not using tags");
405 return kFALSE;
406 }
407 if (fRunNumbers.Length()) {
a8739e8a 408 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 409 arr = fRunNumbers.Tokenize(" ");
410 TIter next(arr);
411 while ((os=(TObjString*)next())) {
412 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
413 if (!gGrid->Cd(path)) {
a8739e8a 414 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
415 continue;
c57f56b7 416 }
417 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
418 TString msg = "\n##### file: ";
419 msg += path;
420 msg += " type: xml_collection;";
421 if (use_tags) msg += " using_tags: Yes";
422 else msg += " using_tags: No";
423 Info("CheckDataType", msg.Data());
319593fb 424 if (fNrunsPerMaster<2) {
425 AddDataFile(path);
426 } else {
427 nruns++;
428 if (((nruns-1)%fNrunsPerMaster) == 0) {
429 schunk = os->GetString();
430 }
431 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
432 schunk += Form("_%s.xml", os->GetString().Data());
433 path = Form("%s/%s", workdir.Data(), schunk.Data());
434 AddDataFile(path);
435 }
c57f56b7 436 }
437 delete arr;
a8739e8a 438 } else {
439 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
440 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
441 path = Form("%s/%d ", fGridDataDir.Data(), irun);
442 if (!gGrid->Cd(path)) {
443 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
444 continue;
445 }
446 path = Form("%s/%d.xml", workdir.Data(),irun);
447 TString msg = "\n##### file: ";
448 msg += path;
449 msg += " type: xml_collection;";
450 if (use_tags) msg += " using_tags: Yes";
451 else msg += " using_tags: No";
452 Info("CheckDataType", msg.Data());
319593fb 453 if (fNrunsPerMaster<2) {
454 AddDataFile(path);
455 } else {
456 nruns++;
457 if (((nruns-1)%fNrunsPerMaster) == 0) {
458 schunk = Form("%d", irun);
459 }
460 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
461 schunk += Form("_%d.xml", irun);
462 path = Form("%s/%s", workdir.Data(), schunk.Data());
463 AddDataFile(path);
464 }
a8739e8a 465 }
c57f56b7 466 }
467 return kTRUE;
468}
469
470//______________________________________________________________________________
471Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
472{
473// Create dataset for the grid data directory + run number.
474 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
475 if (!Connect()) {
476 Error("CreateDataset", "Cannot create dataset with no grid connection");
477 return kFALSE;
478 }
479
480 // Cd workspace
481 CdWork();
482 TString workdir = gGrid->GetHomeDirectory();
483 workdir += fGridWorkingDir;
484
485 // Compose the 'find' command arguments
486 TString command;
487 TString options = "-x collection ";
bb885a9e 488 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 489 TString conditions = "";
490
491 TString file;
492 TString path;
319593fb 493 Int_t nruns = 0;
494 TString schunk;
495 TGridCollection *cbase, *cadd;
a8739e8a 496 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 497 // Several runs
a8739e8a 498 if (fRunNumbers.Length()) {
499 TObjArray *arr = fRunNumbers.Tokenize(" ");
500 TObjString *os;
501 TIter next(arr);
502 while ((os=(TObjString*)next())) {
503 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
504 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
505 else file = Form("%s.xml", os->GetString().Data());
319593fb 506 // If local collection file does not exist, create it via 'find' command.
507 if (gSystem->AccessPathName(file)) {
508 command = "find ";
509 command += options;
510 command += path;
511 command += pattern;
512 command += conditions;
513 TGridResult *res = gGrid->Command(command);
514 if (res) delete res;
515 // Write standard output to file
516 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
517 }
a8739e8a 518 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 519 // Check if there is one run per master job.
520 if (fNrunsPerMaster<2) {
521 if (FileExists(file)) {
522 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
523 continue;
524 }
525 // Copy xml file to alien space
526 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
527 if (!FileExists(file)) {
528 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
529 delete arr;
530 return kFALSE;
531 }
532 } else {
533 nruns++;
534 if (((nruns-1)%fNrunsPerMaster) == 0) {
535 schunk = os->GetString();
536 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
537 } else {
538 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
539 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
540 cbase->Add(cadd);
541 delete cadd;
542 }
543 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
544 continue;
545 }
546 schunk += Form("_%s.xml", os->GetString().Data());
547 if (FileExists(schunk)) {
548 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
549 continue;
550 }
551 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
552 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
553 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
554 if (!FileExists(schunk)) {
555 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
556 delete arr;
557 return kFALSE;
558 }
559 }
a8739e8a 560 }
561 delete arr;
562 } else {
563 // Process a full run range.
564 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
565 path = Form("%s/%d ", fGridDataDir.Data(), irun);
566 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
567 else file = Form("%d.xml", irun);
319593fb 568 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 569 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
570// gGrid->Rm(file);
571 continue;
572 }
319593fb 573 // If local collection file does not exist, create it via 'find' command.
574 if (gSystem->AccessPathName(file)) {
575 command = "find ";
576 command += options;
577 command += path;
578 command += pattern;
579 command += conditions;
580 TGridResult *res = gGrid->Command(command);
581 if (res) delete res;
582 // Write standard output to file
583 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
584 }
a8739e8a 585 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 586 // Check if there is one run per master job.
587 if (fNrunsPerMaster<2) {
588 if (FileExists(file)) {
589 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
590 continue;
591 }
592 // Copy xml file to alien space
593 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
594 if (!FileExists(file)) {
595 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
596 return kFALSE;
597 }
598 } else {
599 nruns++;
600 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
601 if (((nruns-1)%fNrunsPerMaster) == 0) {
602 schunk = Form("%d", irun);
603 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
604 } else {
605 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
606 cbase->Add(cadd);
607 delete cadd;
608 }
609 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
610 continue;
611 }
612 schunk += Form("_%d.xml", irun);
613 if (FileExists(schunk)) {
614 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
615 continue;
616 }
617 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
618 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
619 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
620 if (!FileExists(schunk)) {
621 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
622 return kFALSE;
623 }
624 }
c57f56b7 625 }
a8739e8a 626 }
c57f56b7 627 return kTRUE;
628}
629
630//______________________________________________________________________________
631Bool_t AliAnalysisAlien::CreateJDL()
632{
633// Generate a JDL file according to current settings. The name of the file is
634// specified by fJDLName.
635 Bool_t error = kFALSE;
636 TObjArray *arr = 0;
637 Bool_t copy = kTRUE;
638 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
639 Bool_t generate = kTRUE;
640 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
641 if (!Connect()) {
642 Error("CreateJDL", "Alien connection required");
643 return kFALSE;
644 }
645 // Check validity of alien workspace
646 CdWork();
647 TString workdir = gGrid->GetHomeDirectory();
648 workdir += fGridWorkingDir;
649 if (generate) {
650 TObjString *os;
651 if (!fInputFiles) {
652 Error("CreateJDL()", "Define some input files for your analysis.");
653 error = kTRUE;
654 }
655 // Compose list of input files
656 // Check if output files were defined
657 if (!fOutputFiles.Length()) {
658 Error("CreateJDL", "You must define at least one output file");
659 error = kTRUE;
660 }
661 // Check if an output directory was defined and valid
662 if (!fGridOutputDir.Length()) {
663 Error("CreateJDL", "You must define AliEn output directory");
664 error = kTRUE;
665 } else {
666 if (!gGrid->Cd(fGridOutputDir)) {
667 if (gGrid->Mkdir(fGridOutputDir)) {
668 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
669 } else {
670 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
671 error = kTRUE;
672 }
673 }
674 gGrid->Cd(workdir);
675 }
676 // Exit if any error up to now
677 if (error) return kFALSE;
678 // Set JDL fields
679 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
680 fGridJDL->SetExecutable(fExecutable);
681// fGridJDL->SetTTL((UInt_t)fTTL);
682 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
683 if (fMaxInitFailed > 0)
684 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
685 if (fSplitMaxInputFileNumber > 0)
686 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
687 if (fSplitMode.Length())
688 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
689// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
690 if (fAliROOTVersion.Length())
691 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
692 if (fROOTVersion.Length())
693 fGridJDL->AddToPackages("ROOT", fROOTVersion);
694 if (fAPIVersion.Length())
695 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
696 fGridJDL->SetInputDataListFormat(fInputFormat);
697 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 698 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
699 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
700 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
701 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
702 if (fAdditionalLibs.Length()) {
703 arr = fAdditionalLibs.Tokenize(" ");
704 TIter next(arr);
705 while ((os=(TObjString*)next())) {
706 if (os->GetString().Contains(".so")) continue;
707 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
708 }
709 delete arr;
710 }
4e5c5506 711 if (fPackages) {
712 TIter next(fPackages);
713 TObject *obj;
714 while ((obj=next()))
715 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
716 }
c57f56b7 717 if (fOutputArchive.Length()) {
718 arr = fOutputArchive.Tokenize(" ");
719 TIter next(arr);
720 while ((os=(TObjString*)next()))
43da816a 721 if (!os->GetString().Contains("@") && fCloseSE.Length())
722 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
723 else
724 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 725 delete arr;
726 }
c57f56b7 727 arr = fOutputFiles.Tokenize(" ");
728 TIter next(arr);
43da816a 729 while ((os=(TObjString*)next())) {
730 if (!os->GetString().Contains("@") && fCloseSE.Length())
731 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
732 else
733 fGridJDL->AddToOutputSandbox(os->GetString());
734 }
c57f56b7 735 delete arr;
736// fGridJDL->SetPrice((UInt_t)fPrice);
737 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
738 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
739 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
a8739e8a 740 // Depending if going through a run range or not, generate one or mode jdl's
741 if (!fRunRange[0]) WriteJDL(-1,copy);
742 else {
743 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++)
b15e08c1 744 if (!WriteJDL(irun-fRunRange[0],copy)) break;
c57f56b7 745 }
c57f56b7 746 }
747 // Copy jdl to grid workspace
a8739e8a 748 if (copy) {
c57f56b7 749 if (fAdditionalLibs.Length()) {
e7c71df0 750 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 751 TObjString *os;
752 TIter next(arr);
753 while ((os=(TObjString*)next())) {
c57f56b7 754 if (os->GetString().Contains(".so")) continue;
4e5c5506 755 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 756 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
757 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
758 }
759 delete arr;
760 }
4e5c5506 761 if (fPackages) {
762 TIter next(fPackages);
763 TObject *obj;
764 while ((obj=next())) {
765 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
766 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
767 }
768 }
c57f56b7 769 }
770 return kTRUE;
771}
772
a8739e8a 773//______________________________________________________________________________
774Bool_t AliAnalysisAlien::WriteJDL(Int_t findex, Bool_t copy)
775{
776// Writes one or more JDL's corresponding to findex. If findex is negative,
777// all run numbers are considered in one go (jdl). For non-negative indices
778// they correspond to the indices in the array fInputFiles.
779 if (!fInputFiles) return kFALSE;
780 TObjString *os;
781 TString line;
782 TString workdir = gGrid->GetHomeDirectory();
783 workdir += fGridWorkingDir;
a8739e8a 784 if (findex < 0) {
785 TIter next(fInputFiles);
786 while ((os=(TObjString*)next()))
787 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
788 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
789 } else {
790 os = (TObjString*)fInputFiles->At(findex);
b15e08c1 791 if (!os) return kFALSE;
a8739e8a 792 line = "#Input xml collection\n";
793 line += "InputDataCollection = {";
794 line += Form(" \"LF:%s,nodownload\"", os->GetString().Data());
795 line += "\n};\n";
796 fGridJDL->SetOutputDirectory(Form("%s/%s/%03d#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data(),findex));
797 }
798
799
800 // Generate the JDL as a string
801 TString sjdl = fGridJDL->Generate();
802 Int_t index;
803 index = sjdl.Index("Executable");
804 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
805 index = sjdl.Index("Split ");
806 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
807 index = sjdl.Index("SplitMaxInputFileNumber");
808 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
809 index = sjdl.Index("InputDataCollection");
810 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
811 else {
812 index = sjdl.Index("OutputDir");
813 sjdl.Insert(index, line.Data());
814 }
815 index = sjdl.Index("InputFile");
816 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
817 index = sjdl.Index("InputDataList ");
818 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
819 index = sjdl.Index("InputDataListFormat");
820 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
821 index = sjdl.Index("Price");
822 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
823 index = sjdl.Index("Requirements");
824 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
825 index = sjdl.Index("Packages");
826 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
827 index = sjdl.Index("User =");
828 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
829 index = sjdl.Index("TTL");
830 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
831 index = sjdl.Index("OutputFile");
832 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
833 index = sjdl.Index("OutputDir");
834 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
835 index = sjdl.Index("OutputArchive");
836 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
837 index = sjdl.Index("MaxInitFailed");
838 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
839 index = sjdl.Index("MasterResubmitThreshold");
840 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
841 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
842 index = sjdl.Index("Validationcommand");
843 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
844 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
845 sjdl.ReplaceAll("(member", "\n (member");
846 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
847 sjdl.ReplaceAll("{", "{\n ");
848 sjdl.ReplaceAll("};", "\n};");
849 sjdl.ReplaceAll("{\n \n", "{\n");
850 sjdl.ReplaceAll("\n\n", "\n");
851 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
852 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
853 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
854 index = sjdl.Index("JDLVariables");
855 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
856 // Write jdl to file
857 TString snjdl = fJDLName;
858 if (findex >=0) snjdl.ReplaceAll(".jdl", Form("_%03d.jdl", findex));
859 ofstream out;
860 out.open(snjdl.Data(), ios::out);
861 if (out.bad()) {
862 Error("CreateJDL", "Bad file name: %s", snjdl.Data());
863 return kFALSE;
864 }
865 out << sjdl << endl;
866
867 // Copy jdl to grid workspace
868 if (!copy) {
869 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
870 } else {
871 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", snjdl.Data());
872 if (FileExists(snjdl)) gGrid->Rm(snjdl);
873 TFile::Cp(Form("file:%s",snjdl.Data()), Form("alien://%s/%s", workdir.Data(), snjdl.Data()));
874 }
875 return kTRUE;
876}
877
c57f56b7 878//______________________________________________________________________________
879Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
880{
881// Returns true if file exists.
882 if (!gGrid) {
883 Error("FileExists", "No connection to grid");
884 return kFALSE;
885 }
886 TGridResult *res = gGrid->Ls(lfn);
887 if (!res) return kFALSE;
888 TMap *map = dynamic_cast<TMap*>(res->At(0));
889 if (!map) {
890 delete res;
891 return kFALSE;
892 }
893 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
894 if (!objs || !objs->GetString().Length()) {
895 delete res;
896 return kFALSE;
897 }
898 delete res;
899 return kTRUE;
900}
901
902//______________________________________________________________________________
903void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
904{
905// Check input data type.
906 is_collection = kFALSE;
907 is_xml = kFALSE;
908 use_tags = kFALSE;
909 if (!gGrid) {
910 Error("CheckDataType", "No connection to grid");
911 return;
912 }
913 is_collection = IsCollection(lfn);
914 TString msg = "\n##### file: ";
915 msg += lfn;
916 if (is_collection) {
917 msg += " type: raw_collection;";
918 // special treatment for collections
919 is_xml = kFALSE;
920 // check for tag files in the collection
921 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
922 if (!res) {
923 msg += " using_tags: No (unknown)";
924 Info("CheckDataType", msg.Data());
925 return;
926 }
927 const char* typeStr = res->GetKey(0, "origLFN");
928 if (!typeStr || !strlen(typeStr)) {
929 msg += " using_tags: No (unknown)";
930 Info("CheckDataType", msg.Data());
931 return;
932 }
933 TString file = typeStr;
934 use_tags = file.Contains(".tag");
935 if (use_tags) msg += " using_tags: Yes";
936 else msg += " using_tags: No";
937 Info("CheckDataType", msg.Data());
938 return;
939 }
940 TString slfn(lfn);
941 slfn.ToLower();
942 is_xml = slfn.Contains(".xml");
943 if (is_xml) {
944 // Open xml collection and check if there are tag files inside
945 msg += " type: xml_collection;";
946 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
947 if (!coll) {
948 msg += " using_tags: No (unknown)";
949 Info("CheckDataType", msg.Data());
950 return;
951 }
952 TMap *map = coll->Next();
953 if (!map) {
954 msg += " using_tags: No (unknown)";
955 Info("CheckDataType", msg.Data());
956 return;
957 }
958 map = (TMap*)map->GetValue("");
959 TString file;
960 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
961 use_tags = file.Contains(".tag");
962 delete coll;
963 if (use_tags) msg += " using_tags: Yes";
964 else msg += " using_tags: No";
965 Info("CheckDataType", msg.Data());
966 return;
967 }
968 use_tags = slfn.Contains(".tag");
969 if (slfn.Contains(".root")) msg += " type: root file;";
970 else msg += " type: unhnown file;";
971 if (use_tags) msg += " using_tags: Yes";
972 else msg += " using_tags: No";
973 Info("CheckDataType", msg.Data());
974}
975
4e5c5506 976//______________________________________________________________________________
977void AliAnalysisAlien::EnablePackage(const char *package)
978{
979// Enables a par file supposed to exist in the current directory.
980 TString pkg(package);
981 pkg.ReplaceAll(".par", "");
982 pkg += ".par";
983 if (gSystem->AccessPathName(pkg)) {
984 Error("EnablePackage", "Package %s not found", pkg.Data());
985 return;
986 }
987 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
988 Info("EnablePackage", "AliEn plugin will use .par packages");
989 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
990 if (!fPackages) {
991 fPackages = new TObjArray();
992 fPackages->SetOwner();
993 }
994 fPackages->Add(new TObjString(pkg));
995}
996
c57f56b7 997//______________________________________________________________________________
998Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
999{
1000// Returns true if file is a collection. Functionality duplicated from
1001// TAlien::Type() because we don't want to directly depend on TAlien.
1002 if (!gGrid) {
1003 Error("IsCollection", "No connection to grid");
1004 return kFALSE;
1005 }
1006 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1007 if (!res) return kFALSE;
1008 const char* typeStr = res->GetKey(0, "type");
1009 if (!typeStr || !strlen(typeStr)) return kFALSE;
1010 if (!strcmp(typeStr, "collection")) return kTRUE;
1011 delete res;
1012 return kFALSE;
1013}
1014
16a4353c 1015//______________________________________________________________________________
1016void AliAnalysisAlien::Print(Option_t *) const
1017{
1018// Print current plugin settings.
1019 printf("### AliEn analysis plugin current settings ###\n");
1020 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1021 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1022 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1023 if (fUser.Length())
1024 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1025 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1026 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1027 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1028 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1029 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1030 if (fRunNumbers.Length())
1031 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1032 if (fRunRange[0])
1033 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1034 if (!fRunRange[0] && !fRunNumbers.Length()) {
1035 TIter next(fInputFiles);
1036 TObject *obj;
1037 TString list;
1038 while ((obj=next())) list += obj->GetName();
1039 printf("= Input files to be processed: _________________ %s\n", list.Data());
1040 }
1041 if (TestBit(AliAnalysisGrid::kTest))
1042 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1043 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1044 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1045 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1046 printf("=====================================================================\n");
1047 printf("= Job price: ___________________________________ %d\n", fPrice);
1048 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1049 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1050 if (fMaxInitFailed>0)
1051 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1052 if (fMasterResubmitThreshold>0)
1053 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1054 if (fNrunsPerMaster>0)
1055 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
16a4353c 1056 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1057 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
1058 if (fArguments.Length())
1059 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1060 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1061 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1062 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1063 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1064 if (fDatasetName)
1065 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1066 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1067 if (fIncludePath.Data())
1068 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1069 if (fCloseSE.Length())
1070 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1071 if (fFriendChainName.Length())
1072 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1073 if (fPackages) {
1074 TIter next(fPackages);
1075 TObject *obj;
1076 TString list;
1077 while ((obj=next())) list += obj->GetName();
1078 printf("= Par files to be used: ________________________ %s\n", list.Data());
1079 }
1080}
1081
c57f56b7 1082//______________________________________________________________________________
1083void AliAnalysisAlien::SetDefaults()
1084{
1085// Set default values for everything. What cannot be filled will be left empty.
1086 if (fGridJDL) delete fGridJDL;
1087 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1088 fPrice = 1;
1089 fTTL = 30000;
1090 fSplitMaxInputFileNumber = 100;
1091 fMaxInitFailed = 0;
1092 fMasterResubmitThreshold = 0;
bb885a9e 1093 fNtestFiles = 10;
a8739e8a 1094 fRunRange[0] = 0;
1095 fRunRange[1] = 0;
319593fb 1096 fNrunsPerMaster = 1;
16a4353c 1097 fMaxMergeFiles = 100;
c57f56b7 1098 fRunNumbers = "";
1099 fExecutable = "analysis.sh";
1100 fArguments = "";
1101 fAnalysisMacro = "myAnalysis.C";
1102 fAnalysisSource = "";
1103 fAdditionalLibs = "";
1104 fSplitMode = "se";
1105 fAPIVersion = "";
1106 fROOTVersion = "";
1107 fAliROOTVersion = "";
1108 fUser = ""; // Your alien user name
1109 fGridWorkingDir = "";
1110 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1111 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1112 fFriendChainName = "";
c57f56b7 1113 fGridOutputDir = "output";
1114 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1115 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1116 fInputFormat = "xml-single";
1117 fJDLName = "analysis.jdl";
bb885a9e 1118 fMergeExcludes = "";
c57f56b7 1119}
1120
1121//______________________________________________________________________________
1122Bool_t AliAnalysisAlien::MergeOutputs()
1123{
1124// Merge analysis outputs existing in the AliEn space.
1125 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1126 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1127 if (!Connect()) {
1128 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1129 return kFALSE;
1130 }
1131 // Get the output path
1132 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1133 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
1134 if (!gGrid->Cd(output)) {
1135 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1136 return kFALSE;
1137 }
1138 if (!fOutputFiles.Length()) {
1139 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1140 return kFALSE;
1141 }
1142 TObjArray *list = fOutputFiles.Tokenize(" ");
1143 TIter next(list);
1144 TObjString *str;
1145 TString command;
1146 TString output_file;
16a4353c 1147 TString output_chunk;
1148 TString previous_chunk;
1149 Int_t count_chunk = 0;
1150 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1151 Bool_t merged = kTRUE;
1152 while((str=(TObjString*)next())) {
1153 output_file = str->GetString();
1154 Int_t index = output_file.Index("@");
1155 if (index > 0) output_file.Remove(index);
319593fb 1156 // Skip already merged outputs
1157 if (!gSystem->AccessPathName(output_file)) {
1158 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1159 continue;
1160 }
bb885a9e 1161 if (fMergeExcludes.Length() &&
1162 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1163 // Perform a 'find' command in the output directory, looking for registered outputs
c57f56b7 1164 command = Form("find %s/ *%s", output.Data(), output_file.Data());
1165 printf("command: %s\n", command.Data());
1166 TGridResult *res = gGrid->Command(command);
1167 if (!res) continue;
1168 TFileMerger *fm = 0;
1169 TIter nextmap(res);
1170 TMap *map;
16a4353c 1171 previous_chunk = "";
1172 count_chunk = 0;
319593fb 1173 // Check if there is a merge operation to resume
1174 output_chunk = output_file;
1175 output_chunk.ReplaceAll(".root", "_*.root");
1176 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1177 while (1) {
1178 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1179 if (!map) {
1180 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1181 delete res;
1182 return kFALSE;
1183 }
1184 output_chunk = output_file;
1185 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1186 printf("%s\n", output_chunk.Data());
1187 count_chunk++;
1188 if (gSystem->AccessPathName(output_chunk)) continue;
1189 // Merged file with chunks up to <count_chunk> found
1190 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1191 previous_chunk = output_chunk;
1192 break;
1193 }
1194 }
16a4353c 1195 count_zero = fMaxMergeFiles;
c57f56b7 1196 while ((map=(TMap*)nextmap())) {
16a4353c 1197 // Loop 'find' results and get next LFN
1198 if (count_zero == fMaxMergeFiles) {
1199 // First file in chunk - create file merger and add previous chunk if any.
1200 fm = new TFileMerger(kFALSE);
1201 fm->SetFastMethod(kTRUE);
1202 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1203 output_chunk = output_file;
1204 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1205 }
1206 // If last file found, put merged results in the output file
1207 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1208 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1209 if (!objs || !objs->GetString().Length()) {
16a4353c 1210 // Nothing found - skip this output
c57f56b7 1211 delete res;
16a4353c 1212 delete fm;
1213 break;
1214 }
1215 // Add file to be merged and decrement chunk counter.
1216 fm->AddFile(objs->GetString());
1217 count_zero--;
1218 if (count_zero==0 || map == res->Last()) {
1219 fm->OutputFile(output_chunk);
1220 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1221 // Nothing found - skip this output
1222 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1223 delete res;
1224 delete fm;
1225 break;
1226 }
1227 // Merge the outputs, then go to next chunk
1228 if (!fm->Merge()) {
1229 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1230 delete res;
1231 delete fm;
1232 merged = kFALSE;
1233 break;
1234 } else {
1235 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1236 gSystem->Unlink(previous_chunk);
16a4353c 1237 }
1238 if (map == res->Last()) {
1239 delete res;
1240 delete fm;
1241 break;
1242 }
1243 count_chunk++;
1244 count_zero = fMaxMergeFiles;
1245 previous_chunk = output_chunk;
c57f56b7 1246 }
c57f56b7 1247 }
c57f56b7 1248 }
1249 if (!merged) {
1250 Error("MergeOutputs", "Terminate() will NOT be executed");
1251 }
1252 return merged;
1253}
1254
bb885a9e 1255//______________________________________________________________________________
1256void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1257{
1258// Use the output files connected to output containers from the analysis manager
1259// rather than the files defined by SetOutputFiles
1260 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1261 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1262 analysis manager");
1263 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1264}
1265
c57f56b7 1266//______________________________________________________________________________
1267void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
1268{
1269// Start remote grid analysis.
1270
43da816a 1271 // Check if output files have to be taken from the analysis manager
1272 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1273 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1274 if (!mgr || !mgr->IsInitialized()) {
1275 Error("StartAnalysis", "You need an initialized analysis manager for this");
1276 return;
1277 }
1278 fOutputFiles = "";
1279 TIter next(mgr->GetOutputs());
1280 AliAnalysisDataContainer *output;
1281 while ((output=(AliAnalysisDataContainer*)next())) {
1282 const char *filename = output->GetFileName();
1283 if (!(strcmp(filename, "default"))) {
1284 if (!mgr->GetOutputEventHandler()) continue;
1285 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1286 }
1287 if (fOutputFiles.Length()) fOutputFiles += " ";
1288 fOutputFiles += filename;
1289 }
c07b9ce2 1290 // Add extra files registered to the analysis manager
1291 if (mgr->GetExtraFiles().Length()) {
1292 if (fOutputFiles.Length()) fOutputFiles += " ";
1293 fOutputFiles += mgr->GetExtraFiles();
1294 }
43da816a 1295 }
f7b1cbc2 1296// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1297 if (TestBit(AliAnalysisGrid::kOffline)) {
1298 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1299 \n there nor any job run. You can revise the JDL and analysis \
1300 \n macro then run the same in \"submit\" mode.");
1301 } else if (TestBit(AliAnalysisGrid::kTest)) {
1302 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1303 \n dataset.");
1304 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1305 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1306 \n space and job submitted.");
1307 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1308 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
1309 return;
1310 } else {
1311 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1312 }
1313
1314 if (!Connect()) {
1315 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
1316 return;
16a4353c 1317 }
1318 Print();
c57f56b7 1319 if (!CheckInputData()) {
1320 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
1321 return;
1322 }
1323 CreateDataset(fDataPattern);
1324 WriteAnalysisFile();
1325 WriteAnalysisMacro();
1326 WriteExecutable();
1327 WriteValidationScript();
1328 if (!CreateJDL()) return;
1329 if (TestBit(AliAnalysisGrid::kOffline)) return;
1330 if (TestBit(AliAnalysisGrid::kTest)) {
1331 // Locally testing the analysis
1332 Info("StartAnalysis", "\n_______________________________________________________________________ \
1333 \n Running analysis script in a daughter shell as on a worker node \
1334 \n_______________________________________________________________________");
1335 TObjArray *list = fOutputFiles.Tokenize(" ");
1336 TIter next(list);
1337 TObjString *str;
1338 TString output_file;
1339 while((str=(TObjString*)next())) {
1340 output_file = str->GetString();
1341 Int_t index = output_file.Index("@");
1342 if (index > 0) output_file.Remove(index);
43da816a 1343 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1344 }
1345 delete list;
1346 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1347 gSystem->Exec("bash validate.sh");
1348// gSystem->Exec("cat stdout");
1349 return;
1350 }
a8739e8a 1351 // Submit AliEn job(s)
c57f56b7 1352 CdWork();
a8739e8a 1353 TGridResult *res;
c57f56b7 1354 TString jobID = "";
dd74a515 1355 if (!fRunRange[0]) {
1356 // Submit a given xml or a set of runs
a8739e8a 1357 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1358 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1359 if (res) {
1360 const char *cjobId = res->GetKey(0,"jobId");
1361 if (!cjobId) {
1362 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1363 return;
1364 } else {
1365 Info("StartAnalysis", "\n_______________________________________________________________________ \
1366 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1367 \n_______________________________________________________________________",
1368 fJDLName.Data(), cjobId);
1369 jobID = cjobId;
1370 }
1371 delete res;
1372 }
1373 } else {
dd74a515 1374 // Submit for a range of runs.
a8739e8a 1375 TString sjdl;
1376 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
b15e08c1 1377 if (!fInputFiles->At(irun-fRunRange[0])) break;
a8739e8a 1378 sjdl = fJDLName;
1379 sjdl.ReplaceAll(".jdl", Form("_%03d.jdl", irun-fRunRange[0]));
1380 res = gGrid->Command(Form("submit %s", sjdl.Data()));
1381 printf("*************************** %s\n",Form("submit %s", sjdl.Data()));
1382 if (res) {
1383 const char *cjobId1 = res->GetKey(0,"jobId");
1384 if (!cjobId1) {
1385 Error("StartAnalysis", "Your JDL %s could not be submitted", sjdl.Data());
1386 return;
1387 } else {
1388 Info("StartAnalysis", "\n_______________________________________________________________________ \
1389 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1390 \n_______________________________________________________________________",
1391 sjdl.Data(), cjobId1);
1392 jobID += cjobId1;
1393 jobID += " ";
1394 }
1395 delete res;
1396 }
1397 }
c57f56b7 1398 }
a8739e8a 1399
c57f56b7 1400 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1401 \n You may exit at any time and terminate the job later using the option <terminate> \
1402 \n ##################################################################################", jobID.Data());
bb885a9e 1403 gSystem->Exec("aliensh");
c57f56b7 1404}
1405
1406//______________________________________________________________________________
1407void AliAnalysisAlien::WriteAnalysisFile()
1408{
1409// Write current analysis manager into the file analysis.root
1410 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1411 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1412 if (!mgr || !mgr->IsInitialized()) {
1413 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1414 return;
1415 }
1416 // Check analysis type
1417 TObject *handler;
1418 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1419 handler = (TObject*)mgr->GetInputEventHandler();
1420 if (handler) {
1421 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1422 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1423 }
1424 TDirectory *cdir = gDirectory;
1425 TFile *file = TFile::Open("analysis.root", "RECREATE");
1426 if (file) {
1427 mgr->Write();
1428 delete file;
1429 }
1430 if (cdir) cdir->cd();
1431 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1432 }
1433 Bool_t copy = kTRUE;
1434 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1435 if (copy) {
1436 CdWork();
1437 TString workdir = gGrid->GetHomeDirectory();
1438 workdir += fGridWorkingDir;
1439 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1440 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1441 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1442 }
1443}
1444
1445//______________________________________________________________________________
1446void AliAnalysisAlien::WriteAnalysisMacro()
1447{
1448// Write the analysis macro that will steer the analysis in grid mode.
1449 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1450 ofstream out;
1451 out.open(fAnalysisMacro.Data(), ios::out);
1452 if (!out.good()) {
1453 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1454 return;
1455 }
1456 TString func = fAnalysisMacro;
1457 TString type = "ESD";
1458 TString comment = "// Analysis using ";
1459 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1460 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1461 type = "AOD";
1462 comment += "AOD";
1463 }
0df6ccf2 1464 if (type!="AOD" && fFriendChainName!="") {
1465 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1466 return;
1467 }
c57f56b7 1468 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1469 else comment += " data";
1470 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1471 func.ReplaceAll(".C", "");
1472 out << "void " << func.Data() << "()" << endl;
1473 out << "{" << endl;
1474 out << comment.Data() << endl;
1475 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1476 out << " TStopwatch timer;" << endl;
1477 out << " timer.Start();" << endl << endl;
c57f56b7 1478 out << "// load base root libraries" << endl;
1479 out << " gSystem->Load(\"libTree\");" << endl;
1480 out << " gSystem->Load(\"libGeom\");" << endl;
1481 out << " gSystem->Load(\"libVMC\");" << endl;
1482 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1483 out << "// Load analysis framework libraries" << endl;
4e5c5506 1484 if (!fPackages) {
4e5c5506 1485 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1486 out << " gSystem->Load(\"libESD\");" << endl;
1487 out << " gSystem->Load(\"libAOD\");" << endl;
1488 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1489 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1490 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1491 } else {
4e5c5506 1492 TIter next(fPackages);
1493 TObject *obj;
57377eb5 1494 TString pkgname;
1495 Bool_t hasSTEERBase = kFALSE;
1496 Bool_t hasESD = kFALSE;
1497 Bool_t hasAOD = kFALSE;
1498 Bool_t hasANALYSIS = kFALSE;
1499 Bool_t hasANALYSISalice = kFALSE;
1500 Bool_t hasCORRFW = kFALSE;
1501 while ((obj=next())) {
1502 pkgname = obj->GetName();
4478e6f1 1503 if (pkgname == "STEERBase" ||
1504 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1505 if (pkgname == "ESD" ||
1506 pkgname == "ESD.par") hasESD = kTRUE;
1507 if (pkgname == "AOD" ||
1508 pkgname == "AOD.par") hasAOD = kTRUE;
1509 if (pkgname == "ANALYSIS" ||
1510 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1511 if (pkgname == "ANALYSISalice" ||
1512 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1513 if (pkgname == "CORRFW" ||
1514 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1515 }
1516 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1517 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1518 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1519 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1520 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1521 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1522 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1523 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1524 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1525 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1526 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1527 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1528 out << "// Compile other par packages" << endl;
1529 next.Reset();
fcc9bb6f 1530 while ((obj=next())) {
1531 pkgname = obj->GetName();
4478e6f1 1532 if (pkgname == "STEERBase" ||
1533 pkgname == "STEERBase.par" ||
1534 pkgname == "ESD" ||
1535 pkgname == "ESD.par" ||
1536 pkgname == "AOD" ||
1537 pkgname == "AOD.par" ||
1538 pkgname == "ANALYSIS" ||
1539 pkgname == "ANALYSIS.par" ||
1540 pkgname == "ANALYSISalice" ||
1541 pkgname == "ANALYSISalice.par" ||
1542 pkgname == "CORRFW" ||
1543 pkgname == "CORRFW.par") continue;
4e5c5506 1544 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1545 }
4e5c5506 1546 }
43da816a 1547 out << "// include path" << endl;
1548 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1549 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1550 if (fAdditionalLibs.Length()) {
1551 out << "// Add aditional AliRoot libraries" << endl;
1552 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1553 TIter next(list);
1554 TObjString *str;
1555 while((str=(TObjString*)next())) {
1556 if (str->GetString().Contains(".so"))
1557 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1558 }
1559 if (list) delete list;
1560 }
1561 out << endl;
c57f56b7 1562 out << "// analysis source to be compiled at runtime (if any)" << endl;
1563 if (fAnalysisSource.Length()) {
1564 TObjArray *list = fAnalysisSource.Tokenize(" ");
1565 TIter next(list);
1566 TObjString *str;
1567 while((str=(TObjString*)next())) {
1568 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1569 }
1570 if (list) delete list;
1571 }
1572 out << endl;
1573 out << "// connect to AliEn and make the chain" << endl;
1574 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1575 if (IsUsingTags()) {
1576 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1577 } else {
1578 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1579 }
1580 out << "// read the analysis manager from file" << endl;
1581 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1582 out << " if (!file) return;" << endl;
1583 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1584 out << " AliAnalysisManager *mgr = 0;" << endl;
1585 out << " TKey *key;" << endl;
1586 out << " while ((key=(TKey*)nextkey())) {" << endl;
1587 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1588 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1589 out << " };" << endl;
1590 out << " if (!mgr) {" << endl;
1591 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1592 out << " return;" << endl;
1593 out << " }" << endl << endl;
1594 out << " mgr->PrintStatus();" << endl;
1595 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1596 out << " timer.Stop();" << endl;
1597 out << " timer.Print();" << endl;
c57f56b7 1598 out << "}" << endl << endl;
1599 if (IsUsingTags()) {
1600 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1601 out << "{" << endl;
1602 out << "// Create a chain using tags from the xml file." << endl;
1603 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1604 out << " if (!coll) {" << endl;
1605 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1606 out << " return NULL;" << endl;
1607 out << " }" << endl;
1608 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1609 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1610 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1611 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1612 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1613 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1614 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1615 out << " // Check if the cuts configuration file was provided" << endl;
1616 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1617 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1618 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1619 out << " }" << endl;
0df6ccf2 1620 if (fFriendChainName=="") {
1621 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1622 } else {
1623 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1624 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1625 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1626 }
c57f56b7 1627 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1628 out << " chain->ls();" << endl;
1629 out << " return chain;" << endl;
fcc9bb6f 1630 out << "}" << endl << endl;
c57f56b7 1631 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1632 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1633 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1634 msg += " AliLHCTagCuts *lhcCuts,\n";
1635 msg += " AliDetectorTagCuts *detCuts,\n";
1636 msg += " AliEventTagCuts *evCuts)";
1637 Info("WriteAnalysisMacro", msg.Data());
1638 }
0df6ccf2 1639 }
1640 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1641 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1642 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1643 out << "{" << endl;
1644 out << "// Create a chain using url's from xml file" << endl;
1645 out << " TString treename = type;" << endl;
1646 out << " treename.ToLower();" << endl;
1647 out << " treename += \"Tree\";" << endl;
e02fee64 1648 out << " printf(\"***************************************\\n\");" << endl;
1649 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1650 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1651 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1652 out << " if (!coll) {" << endl;
1653 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1654 out << " return NULL;" << endl;
1655 out << " }" << endl;
1656 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1657 if(fFriendChainName!="") {
1658 out << " TChain *chainFriend = new TChain(treename);" << endl;
1659 }
c57f56b7 1660 out << " coll->Reset();" << endl;
0df6ccf2 1661 out << " while (coll->Next()) {" << endl;
1662 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1663 if(fFriendChainName!="") {
1664 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1665 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1666 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1667 out << " chainFriend->Add(fileFriend.Data());" << endl;
1668 }
1669 out << " }" << endl;
c57f56b7 1670 out << " if (!chain->GetNtrees()) {" << endl;
1671 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1672 out << " return NULL;" << endl;
1673 out << " }" << endl;
0df6ccf2 1674 if(fFriendChainName!="") {
1675 out << " chain->AddFriend(chainFriend);" << endl;
1676 }
c57f56b7 1677 out << " return chain;" << endl;
fcc9bb6f 1678 out << "}" << endl << endl;
c57f56b7 1679 }
4e5c5506 1680 if (fPackages) {
fcc9bb6f 1681 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1682 out << "Bool_t SetupPar(const char *package) {" << endl;
1683 out << "// Compile the package and set it up." << endl;
1684 out << " TString pkgdir = package;" << endl;
1685 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1686 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1687 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1688 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1689 out << " // Check for BUILD.sh and execute" << endl;
1690 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1691 out << " printf(\"*******************************\\n\");" << endl;
1692 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1693 out << " printf(\"*******************************\\n\");" << endl;
1694 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1695 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1696 out << " gSystem->ChangeDirectory(cdir);" << endl;
1697 out << " return kFALSE;" << endl;
1698 out << " }" << endl;
1699 out << " } else {" << endl;
fcc9bb6f 1700 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1701 out << " gSystem->ChangeDirectory(cdir);" << endl;
1702 out << " return kFALSE;" << endl;
1703 out << " }" << endl;
1704 out << " // Check for SETUP.C and execute" << endl;
1705 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1706 out << " printf(\"*******************************\\n\");" << endl;
1707 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1708 out << " printf(\"*******************************\\n\");" << endl;
1709 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1710 out << " } else {" << endl;
fcc9bb6f 1711 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1712 out << " gSystem->ChangeDirectory(cdir);" << endl;
1713 out << " return kFALSE;" << endl;
1714 out << " }" << endl;
1715 out << " // Restore original workdir" << endl;
1716 out << " gSystem->ChangeDirectory(cdir);" << endl;
1717 out << " return kTRUE;" << endl;
1718 out << "}" << endl;
1719 }
c57f56b7 1720 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1721 }
1722 Bool_t copy = kTRUE;
1723 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1724 if (copy) {
1725 CdWork();
1726 TString workdir = gGrid->GetHomeDirectory();
1727 workdir += fGridWorkingDir;
1728 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1729 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1730 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1731 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1732 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1733 }
1734 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1735 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1736 }
1737}
1738
1739//______________________________________________________________________________
1740void AliAnalysisAlien::WriteExecutable()
1741{
1742// Generate the alien executable script.
1743 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1744 ofstream out;
1745 out.open(fExecutable.Data(), ios::out);
1746 if (out.bad()) {
1747 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1748 return;
1749 }
1750 out << "#!/bin/bash" << endl;
1751 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1752 out << "echo \"=========================================\"" << endl;
1753 out << "echo \"############## PATH : ##############\"" << endl;
1754 out << "echo $PATH" << endl;
1755 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1756 out << "echo $LD_LIBRARY_PATH" << endl;
1757 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1758 out << "echo $ROOTSYS" << endl;
1759 out << "echo \"############## which root : ##############\"" << endl;
1760 out << "which root" << endl;
1761 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1762 out << "echo $ALICE_ROOT" << endl;
1763 out << "echo \"############## which aliroot : ##############\"" << endl;
1764 out << "which aliroot" << endl;
1765 out << "echo \"=========================================\"" << endl << endl;
1766// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1767 out << "root -b -q ";
1768 out << fAnalysisMacro.Data() << endl << endl;
1769 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1770 }
1771 Bool_t copy = kTRUE;
1772 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1773 if (copy) {
1774 CdWork();
1775 TString workdir = gGrid->GetHomeDirectory();
1776 workdir += fGridWorkingDir;
1777 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1778 if (FileExists(executable)) gGrid->Rm(executable);
1779 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1780 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1781 }
1782}
1783
1784//______________________________________________________________________________
1785void AliAnalysisAlien::WriteValidationScript()
1786{
1787// Generate the alien validation script.
1788 // Generate the validation script
1789 TObjString *os;
1790 if (!Connect()) {
1791 Error("WriteValidationScript", "Alien connection required");
1792 return;
1793 }
1794 TString out_stream = "";
1795 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1796 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1797 ofstream out;
1798 out.open("validate.sh", ios::out);
1799 out << "#!/bin/bash" << endl;
1800 out << "##################################################" << endl;
1801 out << "validateout=`dirname $0`" << endl;
1802 out << "validatetime=`date`" << endl;
1803 out << "validated=\"0\";" << endl;
1804 out << "error=0" << endl;
1805 out << "if [ -z $validateout ]" << endl;
1806 out << "then" << endl;
1807 out << " validateout=\".\"" << endl;
1808 out << "fi" << endl << endl;
1809 out << "cd $validateout;" << endl;
1810 out << "validateworkdir=`pwd`;" << endl << endl;
1811 out << "echo \"*******************************************************\"" << out_stream << endl;
1812 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1813 out << "" << endl;
1814 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1815 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1816 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1817 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1818 out << "ls -la ./" << out_stream << endl;
1819 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1820 out << "##################################################" << endl;
ebec370a 1821
1822 out << "" << endl;
1823 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
1824 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
1825 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
1826 out << "" << endl;
1827
1828 out << "if [ ! -f stderr ] ; then" << endl;
1829 out << " error=1" << endl;
1830 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
1831 out << " echo \"Error = $error\" " << out_stream << endl;
1832 out << "fi" << endl;
1833
1834 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
1835 out << " error=1" << endl;
1836 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
1837 out << " echo \"$parArch\" " << out_stream << endl;
1838 out << " echo \"Error = $error\" " << out_stream << endl;
1839 out << "fi" << endl;
1840
1841 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
1842 out << " error=1" << endl;
1843 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
1844 out << " echo \"$segViol\" " << out_stream << endl;
1845 out << " echo \"Error = $error\" " << out_stream << endl;
1846 out << "fi" << endl;
1847
1848 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
1849 out << " error=1" << endl;
1850 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
1851 out << " echo \"$segFault\" " << out_stream << endl;
1852 out << " echo \"Error = $error\" " << out_stream << endl;
1853 out << "fi" << endl;
1854
1855 // Part dedicated to the specific analyses running into the train
1856
c57f56b7 1857 TObjArray *arr = fOutputFiles.Tokenize(" ");
1858 TIter next1(arr);
1859 TString output_file;
1860 while ((os=(TObjString*)next1())) {
1861 output_file = os->GetString();
1862 Int_t index = output_file.Index("@");
1863 if (index > 0) output_file.Remove(index);
1864 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1865 out << " error=1" << endl;
1866 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1867 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1868 out << "fi" << endl;
1869 }
1870 delete arr;
1871 out << "if [ $error = 0 ] ; then" << endl;
1872 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1873 out << "fi" << endl;
1874
1875 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1876 out << "echo \"*******************************************************\"" << out_stream << endl;
1877 out << "cd -" << endl;
1878 out << "exit $error" << endl;
1879 }
1880 Bool_t copy = kTRUE;
1881 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1882 if (copy) {
1883 CdWork();
1884 TString workdir = gGrid->GetHomeDirectory();
1885 workdir += fGridWorkingDir;
1886 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1887 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1888 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1889 }
1890}