]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Small printf correction in the analysis macro generated by the alien plugin
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
bb885a9e 35#include "AliVEventHandler.h"
36#include "AliAnalysisDataContainer.h"
c57f56b7 37#include "AliAnalysisAlien.h"
38
39ClassImp(AliAnalysisAlien)
40
41//______________________________________________________________________________
42AliAnalysisAlien::AliAnalysisAlien()
43 :AliAnalysisGrid(),
44 fGridJDL(NULL),
45 fPrice(0),
46 fTTL(0),
47 fSplitMaxInputFileNumber(0),
48 fMaxInitFailed(0),
49 fMasterResubmitThreshold(0),
bb885a9e 50 fNtestFiles(0),
c57f56b7 51 fRunNumbers(),
52 fExecutable(),
53 fArguments(),
54 fAnalysisMacro(),
55 fAnalysisSource(),
56 fAdditionalLibs(),
57 fSplitMode(),
58 fAPIVersion(),
59 fROOTVersion(),
60 fAliROOTVersion(),
61 fUser(),
62 fGridWorkingDir(),
63 fGridDataDir(),
64 fDataPattern(),
65 fGridOutputDir(),
66 fOutputArchive(),
67 fOutputFiles(),
68 fInputFormat(),
e7c71df0 69 fDatasetName(),
c57f56b7 70 fJDLName(),
bb885a9e 71 fMergeExcludes(),
72 fCloseSE(),
4e5c5506 73 fInputFiles(0),
74 fPackages(0)
c57f56b7 75{
76// Dummy ctor.
77 SetDefaults();
78}
79
80//______________________________________________________________________________
81AliAnalysisAlien::AliAnalysisAlien(const char *name)
82 :AliAnalysisGrid(name),
83 fGridJDL(NULL),
84 fPrice(0),
85 fTTL(0),
86 fSplitMaxInputFileNumber(0),
87 fMaxInitFailed(0),
88 fMasterResubmitThreshold(0),
bb885a9e 89 fNtestFiles(0),
c57f56b7 90 fRunNumbers(),
91 fExecutable(),
92 fArguments(),
93 fAnalysisMacro(),
94 fAnalysisSource(),
95 fAdditionalLibs(),
96 fSplitMode(),
97 fAPIVersion(),
98 fROOTVersion(),
99 fAliROOTVersion(),
100 fUser(),
101 fGridWorkingDir(),
102 fGridDataDir(),
103 fDataPattern(),
104 fGridOutputDir(),
105 fOutputArchive(),
106 fOutputFiles(),
107 fInputFormat(),
e7c71df0 108 fDatasetName(),
c57f56b7 109 fJDLName(),
bb885a9e 110 fMergeExcludes(),
111 fCloseSE(),
4e5c5506 112 fInputFiles(0),
113 fPackages(0)
c57f56b7 114{
115// Default ctor.
116 SetDefaults();
117}
118
119//______________________________________________________________________________
120AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
121 :AliAnalysisGrid(other),
122 fGridJDL(NULL),
123 fPrice(other.fPrice),
124 fTTL(other.fTTL),
125 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
126 fMaxInitFailed(other.fMaxInitFailed),
127 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 128 fNtestFiles(other.fNtestFiles),
c57f56b7 129 fRunNumbers(other.fRunNumbers),
130 fExecutable(other.fExecutable),
131 fArguments(other.fArguments),
132 fAnalysisMacro(other.fAnalysisMacro),
133 fAnalysisSource(other.fAnalysisSource),
134 fAdditionalLibs(other.fAdditionalLibs),
135 fSplitMode(other.fSplitMode),
136 fAPIVersion(other.fAPIVersion),
137 fROOTVersion(other.fROOTVersion),
138 fAliROOTVersion(other.fAliROOTVersion),
139 fUser(other.fUser),
140 fGridWorkingDir(other.fGridWorkingDir),
141 fGridDataDir(other.fGridDataDir),
142 fDataPattern(other.fDataPattern),
143 fGridOutputDir(other.fGridOutputDir),
144 fOutputArchive(other.fOutputArchive),
145 fOutputFiles(other.fOutputFiles),
146 fInputFormat(other.fInputFormat),
e7c71df0 147 fDatasetName(other.fDatasetName),
c57f56b7 148 fJDLName(other.fJDLName),
bb885a9e 149 fMergeExcludes(other.fMergeExcludes),
150 fCloseSE(other.fCloseSE),
4e5c5506 151 fInputFiles(0),
152 fPackages(0)
c57f56b7 153{
154// Copy ctor.
155 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
156 if (other.fInputFiles) {
157 fInputFiles = new TObjArray();
158 TIter next(other.fInputFiles);
159 TObject *obj;
160 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
161 fInputFiles->SetOwner();
162 }
4e5c5506 163 if (other.fPackages) {
164 fPackages = new TObjArray();
165 TIter next(other.fPackages);
166 TObject *obj;
167 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
168 fPackages->SetOwner();
169 }
c57f56b7 170}
171
172//______________________________________________________________________________
173AliAnalysisAlien::~AliAnalysisAlien()
174{
175// Destructor.
176 if (fGridJDL) delete fGridJDL;
177 if (fInputFiles) delete fInputFiles;
4e5c5506 178 if (fPackages) delete fPackages;
c57f56b7 179}
180
181//______________________________________________________________________________
182AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
183{
184// Assignment.
185 if (this != &other) {
186 AliAnalysisGrid::operator=(other);
187 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
188 fPrice = other.fPrice;
189 fTTL = other.fTTL;
190 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
191 fMaxInitFailed = other.fMaxInitFailed;
192 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 193 fNtestFiles = other.fNtestFiles;
c57f56b7 194 fRunNumbers = other.fRunNumbers;
195 fExecutable = other.fExecutable;
196 fArguments = other.fArguments;
197 fAnalysisMacro = other.fAnalysisMacro;
198 fAnalysisSource = other.fAnalysisSource;
199 fAdditionalLibs = other.fAdditionalLibs;
200 fSplitMode = other.fSplitMode;
201 fAPIVersion = other.fAPIVersion;
202 fROOTVersion = other.fROOTVersion;
203 fAliROOTVersion = other.fAliROOTVersion;
204 fUser = other.fUser;
205 fGridWorkingDir = other.fGridWorkingDir;
206 fGridDataDir = other.fGridDataDir;
207 fDataPattern = other.fDataPattern;
208 fGridOutputDir = other.fGridOutputDir;
209 fOutputArchive = other.fOutputArchive;
210 fOutputFiles = other.fOutputFiles;
211 fInputFormat = other.fInputFormat;
e7c71df0 212 fDatasetName = other.fDatasetName;
c57f56b7 213 fJDLName = other.fJDLName;
bb885a9e 214 fMergeExcludes = other.fMergeExcludes;
215 fCloseSE = other.fCloseSE;
c57f56b7 216 if (other.fInputFiles) {
217 fInputFiles = new TObjArray();
218 TIter next(other.fInputFiles);
219 TObject *obj;
220 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
221 fInputFiles->SetOwner();
222 }
4e5c5506 223 if (other.fPackages) {
224 fPackages = new TObjArray();
225 TIter next(other.fPackages);
226 TObject *obj;
227 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
228 fPackages->SetOwner();
229 }
c57f56b7 230 }
231 return *this;
232}
233
234//______________________________________________________________________________
235void AliAnalysisAlien::AddRunNumber(Int_t run)
236{
237// Add a run number to the list of runs to be processed.
238 if (fRunNumbers.Length()) fRunNumbers += " ";
239 fRunNumbers += Form("%d", run);
240}
241
242//______________________________________________________________________________
243void AliAnalysisAlien::AddDataFile(const char *lfn)
244{
245// Adds a data file to the input to be analysed. The file should be a valid LFN
246// or point to an existing file in the alien workdir.
247 if (!fInputFiles) fInputFiles = new TObjArray();
248 fInputFiles->Add(new TObjString(lfn));
249}
250
251//______________________________________________________________________________
252Bool_t AliAnalysisAlien::Connect()
253{
254// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
255 if (gGrid && gGrid->IsConnected()) return kTRUE;
256 if (!gSystem->Getenv("alien_API_USER")) {
257 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
258 gSystem->Getenv("UID"));
259 return kFALSE;
260 }
261 if (!gGrid) {
262 Info("Connect", "Trying to connect to AliEn ...");
263 TGrid::Connect("alien://");
264 }
265 if (!gGrid || !gGrid->IsConnected()) {
266 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
267 return kFALSE;
268 }
269 fUser = gGrid->GetUser();
bb885a9e 270 fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 271 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
272 return kTRUE;
273}
274
275//______________________________________________________________________________
276void AliAnalysisAlien::CdWork()
277{
278// Check validity of alien workspace. Create directory if possible.
279 if (!Connect()) {
280 Error("CdWork", "Alien connection required");
281 return;
282 }
283 TString homedir = gGrid->GetHomeDirectory();
284 TString workdir = homedir + fGridWorkingDir;
285 if (!gGrid->Cd(workdir)) {
286 gGrid->Cd(homedir);
287 if (gGrid->Mkdir(workdir)) {
288 gGrid->Cd(fGridWorkingDir);
289 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
290 } else {
291 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
292 workdir.Data(), homedir.Data());
293 fGridWorkingDir = "";
294 }
295 }
296}
297
298//______________________________________________________________________________
299Bool_t AliAnalysisAlien::CheckInputData()
300{
301// Check validity of input data. If necessary, create xml files.
302 if (!fInputFiles && !fRunNumbers.Length()) {
303 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
304 return kFALSE;
305 }
306 // Process declared files
307 Bool_t is_collection = kFALSE;
308 Bool_t is_xml = kFALSE;
309 Bool_t use_tags = kFALSE;
310 Bool_t checked = kFALSE;
311 CdWork();
312 TString file;
313 TString workdir = gGrid->GetHomeDirectory();
314 workdir += fGridWorkingDir;
315 if (fInputFiles) {
316 TObjString *objstr;
317 TIter next(fInputFiles);
318 while ((objstr=(TObjString*)next())) {
319 file = workdir;
320 file += "/";
321 file += objstr->GetString();
322 // Store full lfn path
323 if (FileExists(file)) objstr->SetString(file);
324 else {
325 file = objstr->GetName();
326 if (!FileExists(objstr->GetName())) {
327 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
328 objstr->GetName(), workdir.Data());
329 return kFALSE;
330 }
331 }
332 Bool_t iscoll, isxml, usetags;
333 CheckDataType(file, iscoll, isxml, usetags);
334 if (!checked) {
335 checked = kTRUE;
336 is_collection = iscoll;
337 is_xml = isxml;
338 use_tags = usetags;
339 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
340 } else {
341 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
342 Error("CheckInputData", "Some conflict was found in the types of inputs");
343 return kFALSE;
344 }
345 }
346 }
347 }
348 // Process requested run numbers
349 if (!fRunNumbers.Length()) return kTRUE;
350 // Check validity of alien data directory
351 if (!fGridDataDir.Length()) {
352 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
353 return kFALSE;
354 }
355 if (!gGrid->Cd(fGridDataDir)) {
356 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
357 return kFALSE;
358 }
359 if (is_collection) {
360 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
361 return kFALSE;
362 }
363
364 if (checked && !is_xml) {
365 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
366 return kFALSE;
367 }
368 // Check validity of run number(s)
369 TObjArray *arr;
370 TObjString *os;
371 TString path;
372 if (!checked) {
373 checked = kTRUE;
374 use_tags = fDataPattern.Contains("tag");
375 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
376 }
377 if (use_tags != fDataPattern.Contains("tag")) {
378 Error("CheckInputData", "Cannot mix input files using/not using tags");
379 return kFALSE;
380 }
381 if (fRunNumbers.Length()) {
382 arr = fRunNumbers.Tokenize(" ");
383 TIter next(arr);
384 while ((os=(TObjString*)next())) {
385 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
386 if (!gGrid->Cd(path)) {
387 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
388 return kFALSE;
389 }
390 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
391 TString msg = "\n##### file: ";
392 msg += path;
393 msg += " type: xml_collection;";
394 if (use_tags) msg += " using_tags: Yes";
395 else msg += " using_tags: No";
396 Info("CheckDataType", msg.Data());
397 AddDataFile(path);
398 }
399 delete arr;
400 }
401 return kTRUE;
402}
403
404//______________________________________________________________________________
405Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
406{
407// Create dataset for the grid data directory + run number.
408 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
409 if (!Connect()) {
410 Error("CreateDataset", "Cannot create dataset with no grid connection");
411 return kFALSE;
412 }
413
414 // Cd workspace
415 CdWork();
416 TString workdir = gGrid->GetHomeDirectory();
417 workdir += fGridWorkingDir;
418
419 // Compose the 'find' command arguments
420 TString command;
421 TString options = "-x collection ";
bb885a9e 422 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 423 TString conditions = "";
424
425 TString file;
426 TString path;
427 if (!fRunNumbers.Length()) return kTRUE;
428 // Several runs
429 TObjArray *arr = fRunNumbers.Tokenize(" ");
430 TObjString *os;
431 TIter next(arr);
432 while ((os=(TObjString*)next())) {
433 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
1535db97 434 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
435 else file = Form("%s.xml", os->GetString().Data());
436 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
c57f56b7 437 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
438 gGrid->Rm(file);
439 }
440 command = "find ";
441 command += options;
442 command += path;
443 command += pattern;
1535db97 444// conditions = Form(" > %s", file.Data());
c57f56b7 445 command += conditions;
446 TGridResult *res = gGrid->Command(command);
447 if (res) delete res;
1535db97 448 // Write standard output to file
449 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
450 if (TestBit(AliAnalysisGrid::kTest)) break;
451 // Copy xml file to alien space
452 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
c57f56b7 453 if (!FileExists(file)) {
454 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
455 delete arr;
456 return kFALSE;
457 }
c57f56b7 458 }
459 delete arr;
c57f56b7 460 return kTRUE;
461}
462
463//______________________________________________________________________________
464Bool_t AliAnalysisAlien::CreateJDL()
465{
466// Generate a JDL file according to current settings. The name of the file is
467// specified by fJDLName.
468 Bool_t error = kFALSE;
469 TObjArray *arr = 0;
470 Bool_t copy = kTRUE;
471 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
472 Bool_t generate = kTRUE;
473 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
474 if (!Connect()) {
475 Error("CreateJDL", "Alien connection required");
476 return kFALSE;
477 }
478 // Check validity of alien workspace
479 CdWork();
480 TString workdir = gGrid->GetHomeDirectory();
481 workdir += fGridWorkingDir;
482 if (generate) {
483 TObjString *os;
484 if (!fInputFiles) {
485 Error("CreateJDL()", "Define some input files for your analysis.");
486 error = kTRUE;
487 }
488 // Compose list of input files
489 // Check if output files were defined
490 if (!fOutputFiles.Length()) {
491 Error("CreateJDL", "You must define at least one output file");
492 error = kTRUE;
493 }
494 // Check if an output directory was defined and valid
495 if (!fGridOutputDir.Length()) {
496 Error("CreateJDL", "You must define AliEn output directory");
497 error = kTRUE;
498 } else {
499 if (!gGrid->Cd(fGridOutputDir)) {
500 if (gGrid->Mkdir(fGridOutputDir)) {
501 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
502 } else {
503 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
504 error = kTRUE;
505 }
506 }
507 gGrid->Cd(workdir);
508 }
509 // Exit if any error up to now
510 if (error) return kFALSE;
511 // Set JDL fields
512 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
513 fGridJDL->SetExecutable(fExecutable);
514// fGridJDL->SetTTL((UInt_t)fTTL);
515 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
516 if (fMaxInitFailed > 0)
517 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
518 if (fSplitMaxInputFileNumber > 0)
519 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
520 if (fSplitMode.Length())
521 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
522// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
523 if (fAliROOTVersion.Length())
524 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
525 if (fROOTVersion.Length())
526 fGridJDL->AddToPackages("ROOT", fROOTVersion);
527 if (fAPIVersion.Length())
528 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
529 fGridJDL->SetInputDataListFormat(fInputFormat);
530 fGridJDL->SetInputDataList("wn.xml");
531 if (fInputFiles) {
532 TIter next(fInputFiles);
533 while ((os=(TObjString*)next()))
534 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
535 }
536 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
537 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
538 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
539 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
540 if (fAdditionalLibs.Length()) {
541 arr = fAdditionalLibs.Tokenize(" ");
542 TIter next(arr);
543 while ((os=(TObjString*)next())) {
544 if (os->GetString().Contains(".so")) continue;
545 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
546 }
547 delete arr;
548 }
4e5c5506 549 if (fPackages) {
550 TIter next(fPackages);
551 TObject *obj;
552 while ((obj=next()))
553 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
554 }
c57f56b7 555 if (fOutputArchive.Length()) {
556 arr = fOutputArchive.Tokenize(" ");
557 TIter next(arr);
558 while ((os=(TObjString*)next()))
559 fGridJDL->AddToOutputArchive(os->GetString().Data());
560 delete arr;
561 }
562 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
563 arr = fOutputFiles.Tokenize(" ");
564 TIter next(arr);
565 while ((os=(TObjString*)next())) fGridJDL->AddToOutputSandbox(os->GetString());
566 delete arr;
567// fGridJDL->SetPrice((UInt_t)fPrice);
568 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
569 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
570 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
571 // Generate the JDL as a string
572 TString sjdl = fGridJDL->Generate();
573 Int_t index;
574 index = sjdl.Index("Executable");
575 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
576 index = sjdl.Index("Split ");
577 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
578 index = sjdl.Index("SplitMaxInputFileNumber");
579 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
580 index = sjdl.Index("InputDataCollection");
581 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
582 index = sjdl.Index("InputFile");
583 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
584 index = sjdl.Index("InputDataList ");
585 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
586 index = sjdl.Index("InputDataListFormat");
587 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
588 index = sjdl.Index("Price");
589 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
590 index = sjdl.Index("Requirements");
591 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
592 index = sjdl.Index("Packages");
593 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
594 index = sjdl.Index("User");
595 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
596 index = sjdl.Index("TTL");
597 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
598 index = sjdl.Index("OutputFile");
599 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
600 index = sjdl.Index("OutputDir");
601 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
602 index = sjdl.Index("OutputArchive");
603 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
604 index = sjdl.Index("MaxInitFailed");
605 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
606 index = sjdl.Index("MasterResubmitThreshold");
607 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
608 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
609 index = sjdl.Index("Validationcommand");
610 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
611 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
612 sjdl.ReplaceAll("(member", "\n (member");
613 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
614 sjdl.ReplaceAll("{", "{\n ");
615 sjdl.ReplaceAll("};", "\n};");
616 sjdl.ReplaceAll("{\n \n", "{\n");
617 sjdl.ReplaceAll("\n\n", "\n");
618 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
619 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
620 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
621 index = sjdl.Index("JDLVariables");
622 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
623 // Write jdl to file
624 ofstream out;
625 out.open(fJDLName.Data(), ios::out);
626 if (out.bad()) {
627 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
628 return kFALSE;
629 }
630 out << sjdl << endl;
631 }
632 // Copy jdl to grid workspace
633 if (!copy) {
634 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
635 } else {
636 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
637 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
638 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
639 if (fAdditionalLibs.Length()) {
e7c71df0 640 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 641 TObjString *os;
642 TIter next(arr);
643 while ((os=(TObjString*)next())) {
c57f56b7 644 if (os->GetString().Contains(".so")) continue;
4e5c5506 645 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 646 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
647 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
648 }
649 delete arr;
650 }
4e5c5506 651 if (fPackages) {
652 TIter next(fPackages);
653 TObject *obj;
654 while ((obj=next())) {
655 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
656 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
657 }
658 }
c57f56b7 659 }
660 return kTRUE;
661}
662
663//______________________________________________________________________________
664Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
665{
666// Returns true if file exists.
667 if (!gGrid) {
668 Error("FileExists", "No connection to grid");
669 return kFALSE;
670 }
671 TGridResult *res = gGrid->Ls(lfn);
672 if (!res) return kFALSE;
673 TMap *map = dynamic_cast<TMap*>(res->At(0));
674 if (!map) {
675 delete res;
676 return kFALSE;
677 }
678 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
679 if (!objs || !objs->GetString().Length()) {
680 delete res;
681 return kFALSE;
682 }
683 delete res;
684 return kTRUE;
685}
686
687//______________________________________________________________________________
688void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
689{
690// Check input data type.
691 is_collection = kFALSE;
692 is_xml = kFALSE;
693 use_tags = kFALSE;
694 if (!gGrid) {
695 Error("CheckDataType", "No connection to grid");
696 return;
697 }
698 is_collection = IsCollection(lfn);
699 TString msg = "\n##### file: ";
700 msg += lfn;
701 if (is_collection) {
702 msg += " type: raw_collection;";
703 // special treatment for collections
704 is_xml = kFALSE;
705 // check for tag files in the collection
706 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
707 if (!res) {
708 msg += " using_tags: No (unknown)";
709 Info("CheckDataType", msg.Data());
710 return;
711 }
712 const char* typeStr = res->GetKey(0, "origLFN");
713 if (!typeStr || !strlen(typeStr)) {
714 msg += " using_tags: No (unknown)";
715 Info("CheckDataType", msg.Data());
716 return;
717 }
718 TString file = typeStr;
719 use_tags = file.Contains(".tag");
720 if (use_tags) msg += " using_tags: Yes";
721 else msg += " using_tags: No";
722 Info("CheckDataType", msg.Data());
723 return;
724 }
725 TString slfn(lfn);
726 slfn.ToLower();
727 is_xml = slfn.Contains(".xml");
728 if (is_xml) {
729 // Open xml collection and check if there are tag files inside
730 msg += " type: xml_collection;";
731 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
732 if (!coll) {
733 msg += " using_tags: No (unknown)";
734 Info("CheckDataType", msg.Data());
735 return;
736 }
737 TMap *map = coll->Next();
738 if (!map) {
739 msg += " using_tags: No (unknown)";
740 Info("CheckDataType", msg.Data());
741 return;
742 }
743 map = (TMap*)map->GetValue("");
744 TString file;
745 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
746 use_tags = file.Contains(".tag");
747 delete coll;
748 if (use_tags) msg += " using_tags: Yes";
749 else msg += " using_tags: No";
750 Info("CheckDataType", msg.Data());
751 return;
752 }
753 use_tags = slfn.Contains(".tag");
754 if (slfn.Contains(".root")) msg += " type: root file;";
755 else msg += " type: unhnown file;";
756 if (use_tags) msg += " using_tags: Yes";
757 else msg += " using_tags: No";
758 Info("CheckDataType", msg.Data());
759}
760
4e5c5506 761//______________________________________________________________________________
762void AliAnalysisAlien::EnablePackage(const char *package)
763{
764// Enables a par file supposed to exist in the current directory.
765 TString pkg(package);
766 pkg.ReplaceAll(".par", "");
767 pkg += ".par";
768 if (gSystem->AccessPathName(pkg)) {
769 Error("EnablePackage", "Package %s not found", pkg.Data());
770 return;
771 }
772 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
773 Info("EnablePackage", "AliEn plugin will use .par packages");
774 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
775 if (!fPackages) {
776 fPackages = new TObjArray();
777 fPackages->SetOwner();
778 }
779 fPackages->Add(new TObjString(pkg));
780}
781
c57f56b7 782//______________________________________________________________________________
783Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
784{
785// Returns true if file is a collection. Functionality duplicated from
786// TAlien::Type() because we don't want to directly depend on TAlien.
787 if (!gGrid) {
788 Error("IsCollection", "No connection to grid");
789 return kFALSE;
790 }
791 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
792 if (!res) return kFALSE;
793 const char* typeStr = res->GetKey(0, "type");
794 if (!typeStr || !strlen(typeStr)) return kFALSE;
795 if (!strcmp(typeStr, "collection")) return kTRUE;
796 delete res;
797 return kFALSE;
798}
799
800//______________________________________________________________________________
801void AliAnalysisAlien::SetDefaults()
802{
803// Set default values for everything. What cannot be filled will be left empty.
804 if (fGridJDL) delete fGridJDL;
805 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
806 fPrice = 1;
807 fTTL = 30000;
808 fSplitMaxInputFileNumber = 100;
809 fMaxInitFailed = 0;
810 fMasterResubmitThreshold = 0;
bb885a9e 811 fNtestFiles = 10;
c57f56b7 812 fRunNumbers = "";
813 fExecutable = "analysis.sh";
814 fArguments = "";
815 fAnalysisMacro = "myAnalysis.C";
816 fAnalysisSource = "";
817 fAdditionalLibs = "";
818 fSplitMode = "se";
819 fAPIVersion = "";
820 fROOTVersion = "";
821 fAliROOTVersion = "";
822 fUser = ""; // Your alien user name
823 fGridWorkingDir = "";
824 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
825 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
826 fGridOutputDir = "output";
827 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
828 fOutputFiles = ""; // Like "AliAODs.root histos.root"
829 fInputFormat = "xml-single";
830 fJDLName = "analysis.jdl";
bb885a9e 831 fMergeExcludes = "";
c57f56b7 832}
833
834//______________________________________________________________________________
835Bool_t AliAnalysisAlien::MergeOutputs()
836{
837// Merge analysis outputs existing in the AliEn space.
838 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
839 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
840 if (!Connect()) {
841 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
842 return kFALSE;
843 }
844 // Get the output path
845 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
846 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
847 if (!gGrid->Cd(output)) {
848 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
849 return kFALSE;
850 }
851 if (!fOutputFiles.Length()) {
852 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
853 return kFALSE;
854 }
855 TObjArray *list = fOutputFiles.Tokenize(" ");
856 TIter next(list);
857 TObjString *str;
858 TString command;
859 TString output_file;
860 Bool_t merged = kTRUE;
861 while((str=(TObjString*)next())) {
862 output_file = str->GetString();
863 Int_t index = output_file.Index("@");
864 if (index > 0) output_file.Remove(index);
bb885a9e 865 if (fMergeExcludes.Length() &&
866 fMergeExcludes.Contains(output_file.Data())) continue;
c57f56b7 867 command = Form("find %s/ *%s", output.Data(), output_file.Data());
868 printf("command: %s\n", command.Data());
869 TGridResult *res = gGrid->Command(command);
870 if (!res) continue;
871 TFileMerger *fm = 0;
872 TIter nextmap(res);
873 TMap *map;
874 while ((map=(TMap*)nextmap())) {
875 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
876 if (!objs || !objs->GetString().Length()) {
877 delete res;
878 continue;
879 }
880 if (!fm) {
881 fm = new TFileMerger(kFALSE);
882 fm->SetFastMethod(kTRUE);
883 fm->OutputFile(output_file);
884 }
885 fm->AddFile(objs->GetString());
886 }
887 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
888 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
889 merged = kFALSE;
890 delete res;
891 continue;
892 }
893 if (!fm->Merge()) {
894 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
895 merged = kFALSE;
896 } else {
897 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
898 }
899 delete fm;
900 delete res;
901 }
902 if (!merged) {
903 Error("MergeOutputs", "Terminate() will NOT be executed");
904 }
905 return merged;
906}
907
bb885a9e 908//______________________________________________________________________________
909void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
910{
911// Use the output files connected to output containers from the analysis manager
912// rather than the files defined by SetOutputFiles
913 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
914 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
915 analysis manager");
916 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
917}
918
c57f56b7 919//______________________________________________________________________________
920void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
921{
922// Start remote grid analysis.
923
924 if (TestBit(AliAnalysisGrid::kOffline)) {
925 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
926 \n there nor any job run. You can revise the JDL and analysis \
927 \n macro then run the same in \"submit\" mode.");
928 } else if (TestBit(AliAnalysisGrid::kTest)) {
929 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
930 \n dataset.");
931 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
932 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
933 \n space and job submitted.");
934 } else if (TestBit(AliAnalysisGrid::kMerge)) {
935 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
936 return;
937 } else {
938 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
939 }
940
941 if (!Connect()) {
942 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
943 return;
944 }
945 if (!CheckInputData()) {
946 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
947 return;
948 }
949 CreateDataset(fDataPattern);
bb885a9e 950 // Check if output files have to be taken from the analysis manager
951 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
952 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
953 if (!mgr || !mgr->IsInitialized()) {
954 Error("StartAnalysis", "You need an initialized analysis manager for this");
955 return;
956 }
957 fOutputFiles = "";
958 TIter next(mgr->GetOutputs());
959 AliAnalysisDataContainer *output;
960 while ((output=(AliAnalysisDataContainer*)next())) {
961 const char *filename = output->GetFileName();
962 if (!(strcmp(filename, "default"))) {
963 if (!mgr->GetOutputEventHandler()) continue;
964 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
965 }
966 if (fOutputFiles.Length()) fOutputFiles += " ";
967 fOutputFiles += filename;
968 }
969 }
c57f56b7 970 WriteAnalysisFile();
971 WriteAnalysisMacro();
972 WriteExecutable();
973 WriteValidationScript();
974 if (!CreateJDL()) return;
975 if (TestBit(AliAnalysisGrid::kOffline)) return;
976 if (TestBit(AliAnalysisGrid::kTest)) {
977 // Locally testing the analysis
978 Info("StartAnalysis", "\n_______________________________________________________________________ \
979 \n Running analysis script in a daughter shell as on a worker node \
980 \n_______________________________________________________________________");
981 TObjArray *list = fOutputFiles.Tokenize(" ");
982 TIter next(list);
983 TObjString *str;
984 TString output_file;
985 while((str=(TObjString*)next())) {
986 output_file = str->GetString();
987 Int_t index = output_file.Index("@");
988 if (index > 0) output_file.Remove(index);
989 gSystem->Exec(Form("rm %s", output_file.Data()));
990 }
991 delete list;
992 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
993 gSystem->Exec("bash validate.sh");
994// gSystem->Exec("cat stdout");
995 return;
996 }
997 // Submit AliEn job
998 CdWork();
999 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1000 TString jobID = "";
1001 if (res) {
1002 const char *cjobId = res->GetKey(0,"jobId");
1003 if (!cjobId) {
1004 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1005 return;
1006 } else {
1007 Info("StartAnalysis", "\n_______________________________________________________________________ \
1008 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1009 \n_______________________________________________________________________",
1010 fJDLName.Data(), cjobId);
1011 jobID = cjobId;
1012 }
1013 delete res;
1014 }
1015 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1016 \n You may exit at any time and terminate the job later using the option <terminate> \
1017 \n ##################################################################################", jobID.Data());
bb885a9e 1018 //gGrid->Shell();
1019 gSystem->Exec("aliensh");
c57f56b7 1020}
1021
1022//______________________________________________________________________________
1023void AliAnalysisAlien::WriteAnalysisFile()
1024{
1025// Write current analysis manager into the file analysis.root
1026 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1027 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1028 if (!mgr || !mgr->IsInitialized()) {
1029 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1030 return;
1031 }
1032 // Check analysis type
1033 TObject *handler;
1034 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1035 handler = (TObject*)mgr->GetInputEventHandler();
1036 if (handler) {
1037 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1038 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1039 }
1040 TDirectory *cdir = gDirectory;
1041 TFile *file = TFile::Open("analysis.root", "RECREATE");
1042 if (file) {
1043 mgr->Write();
1044 delete file;
1045 }
1046 if (cdir) cdir->cd();
1047 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1048 }
1049 Bool_t copy = kTRUE;
1050 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1051 if (copy) {
1052 CdWork();
1053 TString workdir = gGrid->GetHomeDirectory();
1054 workdir += fGridWorkingDir;
1055 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1056 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1057 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1058 }
1059}
1060
1061//______________________________________________________________________________
1062void AliAnalysisAlien::WriteAnalysisMacro()
1063{
1064// Write the analysis macro that will steer the analysis in grid mode.
1065 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1066 ofstream out;
1067 out.open(fAnalysisMacro.Data(), ios::out);
1068 if (!out.good()) {
1069 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1070 return;
1071 }
1072 TString func = fAnalysisMacro;
1073 TString type = "ESD";
1074 TString comment = "// Analysis using ";
1075 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1076 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1077 type = "AOD";
1078 comment += "AOD";
1079 }
1080 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1081 else comment += " data";
1082 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1083 func.ReplaceAll(".C", "");
1084 out << "void " << func.Data() << "()" << endl;
1085 out << "{" << endl;
1086 out << comment.Data() << endl;
1087 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1088 out << "// load base root libraries" << endl;
1089 out << " gSystem->Load(\"libTree\");" << endl;
1090 out << " gSystem->Load(\"libGeom\");" << endl;
1091 out << " gSystem->Load(\"libVMC\");" << endl;
1092 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4e5c5506 1093 if (!fPackages) {
1094 out << "// Load analysis framework libraries" << endl;
1095 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1096 out << " gSystem->Load(\"libESD\");" << endl;
1097 out << " gSystem->Load(\"libAOD\");" << endl;
1098 out << " gSystem->Load(\"libANALYSIS\");" << endl;
1099 out << " gSystem->Load(\"libANALYSISalice\");" << endl << endl;
1100 out << "// include path (remove if using par files)" << endl;
1101 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl << endl;
1102 } else {
1103 out << "// Compile all par packages" << endl;
1104 TIter next(fPackages);
1105 TObject *obj;
1106 while ((obj=next()))
1107 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1108 }
6da75e0b 1109 if (fAdditionalLibs.Length()) {
1110 out << "// Add aditional AliRoot libraries" << endl;
1111 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1112 TIter next(list);
1113 TObjString *str;
1114 while((str=(TObjString*)next())) {
1115 if (str->GetString().Contains(".so"))
1116 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1117 }
1118 if (list) delete list;
1119 }
1120 out << endl;
c57f56b7 1121 out << "// analysis source to be compiled at runtime (if any)" << endl;
1122 if (fAnalysisSource.Length()) {
1123 TObjArray *list = fAnalysisSource.Tokenize(" ");
1124 TIter next(list);
1125 TObjString *str;
1126 while((str=(TObjString*)next())) {
1127 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1128 }
1129 if (list) delete list;
1130 }
1131 out << endl;
1132 out << "// connect to AliEn and make the chain" << endl;
1133 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1134 if (IsUsingTags()) {
1135 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1136 } else {
1137 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1138 }
1139 out << "// read the analysis manager from file" << endl;
1140 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1141 out << " if (!file) return;" << endl;
1142 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1143 out << " AliAnalysisManager *mgr = 0;" << endl;
1144 out << " TKey *key;" << endl;
1145 out << " while ((key=(TKey*)nextkey())) {" << endl;
1146 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1147 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1148 out << " };" << endl;
1149 out << " if (!mgr) {" << endl;
1150 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1151 out << " return;" << endl;
1152 out << " }" << endl << endl;
1153 out << " mgr->PrintStatus();" << endl;
1154 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1155 out << "}" << endl << endl;
1156 if (IsUsingTags()) {
1157 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1158 out << "{" << endl;
1159 out << "// Create a chain using tags from the xml file." << endl;
1160 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1161 out << " if (!coll) {" << endl;
1162 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1163 out << " return NULL;" << endl;
1164 out << " }" << endl;
1165 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1166 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1167 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1168 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1169 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1170 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1171 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1172 out << " // Check if the cuts configuration file was provided" << endl;
1173 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1174 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1175 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1176 out << " }" << endl;
1177 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1178 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1179 out << " chain->ls();" << endl;
1180 out << " return chain;" << endl;
1181 out << "}" << endl;
1182 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1183 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1184 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1185 msg += " AliLHCTagCuts *lhcCuts,\n";
1186 msg += " AliDetectorTagCuts *detCuts,\n";
1187 msg += " AliEventTagCuts *evCuts)";
1188 Info("WriteAnalysisMacro", msg.Data());
1189 }
1190 } else {
1191 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1192 out << "{" << endl;
1193 out << "// Create a chain using url's from xml file" << endl;
1194 out << " TString treename = type;" << endl;
1195 out << " treename.ToLower();" << endl;
1196 out << " treename += \"Tree\";" << endl;
e02fee64 1197 out << " printf(\"***************************************\\n\");" << endl;
1198 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1199 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1200 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1201 out << " if (!coll) {" << endl;
1202 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1203 out << " return NULL;" << endl;
1204 out << " }" << endl;
1205 out << " TChain *chain = new TChain(treename);" << endl;
1206 out << " coll->Reset();" << endl;
1207 out << " while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
1208 out << " if (!chain->GetNtrees()) {" << endl;
1209 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1210 out << " return NULL;" << endl;
1211 out << " }" << endl;
1212 out << " return chain;" << endl;
1213 out << "}" << endl;
1214 }
4e5c5506 1215 if (fPackages) {
1216 out << "Bool_t SetupPar(const char *package) {" << endl;
1217 out << "// Compile the package and set it up." << endl;
1218 out << " TString pkgdir = package;" << endl;
1219 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
1220 out << " gSystem->Exec(Form(\"tar xvzf %s\", package));" << endl;
1221 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1222 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1223 out << " // Check for BUILD.sh and execute" << endl;
1224 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1225 out << " printf(\"*******************************\\n\");" << endl;
1226 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1227 out << " printf(\"*******************************\\n\");" << endl;
1228 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
1229 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", package);" << endl;
1230 out << " gSystem->ChangeDirectory(cdir);" << endl;
1231 out << " return kFALSE;" << endl;
1232 out << " }" << endl;
1233 out << " } else {" << endl;
1234 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", package);" << endl;
1235 out << " gSystem->ChangeDirectory(cdir);" << endl;
1236 out << " return kFALSE;" << endl;
1237 out << " }" << endl;
1238 out << " // Check for SETUP.C and execute" << endl;
1239 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1240 out << " printf(\"*******************************\\n\");" << endl;
1241 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1242 out << " printf(\"*******************************\\n\");" << endl;
1243 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1244 out << " } else {" << endl;
1245 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", package);" << endl;
1246 out << " gSystem->ChangeDirectory(cdir);" << endl;
1247 out << " return kFALSE;" << endl;
1248 out << " }" << endl;
1249 out << " // Restore original workdir" << endl;
1250 out << " gSystem->ChangeDirectory(cdir);" << endl;
1251 out << " return kTRUE;" << endl;
1252 out << "}" << endl;
1253 }
c57f56b7 1254 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1255 }
1256 Bool_t copy = kTRUE;
1257 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1258 if (copy) {
1259 CdWork();
1260 TString workdir = gGrid->GetHomeDirectory();
1261 workdir += fGridWorkingDir;
1262 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1263 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1264 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1265 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1266 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1267 }
1268 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1269 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1270 }
1271}
1272
1273//______________________________________________________________________________
1274void AliAnalysisAlien::WriteExecutable()
1275{
1276// Generate the alien executable script.
1277 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1278 ofstream out;
1279 out.open(fExecutable.Data(), ios::out);
1280 if (out.bad()) {
1281 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1282 return;
1283 }
1284 out << "#!/bin/bash" << endl;
1285 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1286 out << "echo \"=========================================\"" << endl;
1287 out << "echo \"############## PATH : ##############\"" << endl;
1288 out << "echo $PATH" << endl;
1289 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1290 out << "echo $LD_LIBRARY_PATH" << endl;
1291 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1292 out << "echo $ROOTSYS" << endl;
1293 out << "echo \"############## which root : ##############\"" << endl;
1294 out << "which root" << endl;
1295 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1296 out << "echo $ALICE_ROOT" << endl;
1297 out << "echo \"############## which aliroot : ##############\"" << endl;
1298 out << "which aliroot" << endl;
1299 out << "echo \"=========================================\"" << endl << endl;
1300// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1301 out << "root -b -q ";
1302 out << fAnalysisMacro.Data() << endl << endl;
1303 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1304 }
1305 Bool_t copy = kTRUE;
1306 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1307 if (copy) {
1308 CdWork();
1309 TString workdir = gGrid->GetHomeDirectory();
1310 workdir += fGridWorkingDir;
1311 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1312 if (FileExists(executable)) gGrid->Rm(executable);
1313 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1314 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1315 }
1316}
1317
1318//______________________________________________________________________________
1319void AliAnalysisAlien::WriteValidationScript()
1320{
1321// Generate the alien validation script.
1322 // Generate the validation script
1323 TObjString *os;
1324 if (!Connect()) {
1325 Error("WriteValidationScript", "Alien connection required");
1326 return;
1327 }
1328 TString out_stream = "";
1329 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1330 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1331 ofstream out;
1332 out.open("validate.sh", ios::out);
1333 out << "#!/bin/bash" << endl;
1334 out << "##################################################" << endl;
1335 out << "validateout=`dirname $0`" << endl;
1336 out << "validatetime=`date`" << endl;
1337 out << "validated=\"0\";" << endl;
1338 out << "error=0" << endl;
1339 out << "if [ -z $validateout ]" << endl;
1340 out << "then" << endl;
1341 out << " validateout=\".\"" << endl;
1342 out << "fi" << endl << endl;
1343 out << "cd $validateout;" << endl;
1344 out << "validateworkdir=`pwd`;" << endl << endl;
1345 out << "echo \"*******************************************************\"" << out_stream << endl;
1346 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1347 out << "" << endl;
1348 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1349 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1350 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1351 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1352 out << "ls -la ./" << out_stream << endl;
1353 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1354 out << "##################################################" << endl;
1355 TObjArray *arr = fOutputFiles.Tokenize(" ");
1356 TIter next1(arr);
1357 TString output_file;
1358 while ((os=(TObjString*)next1())) {
1359 output_file = os->GetString();
1360 Int_t index = output_file.Index("@");
1361 if (index > 0) output_file.Remove(index);
1362 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1363 out << " error=1" << endl;
1364 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1365 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1366 out << "fi" << endl;
1367 }
1368 delete arr;
1369 out << "if [ $error = 0 ] ; then" << endl;
1370 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1371 out << "fi" << endl;
1372
1373 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1374 out << "echo \"*******************************************************\"" << out_stream << endl;
1375 out << "cd -" << endl;
1376 out << "exit $error" << endl;
1377 }
1378 Bool_t copy = kTRUE;
1379 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1380 if (copy) {
1381 CdWork();
1382 TString workdir = gGrid->GetHomeDirectory();
1383 workdir += fGridWorkingDir;
1384 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1385 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1386 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1387 }
1388}