]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Analysis train macro. Can run locally, in proof or in grid mode (using the AliEn...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
bb885a9e 35#include "AliVEventHandler.h"
36#include "AliAnalysisDataContainer.h"
c57f56b7 37#include "AliAnalysisAlien.h"
38
39ClassImp(AliAnalysisAlien)
40
41//______________________________________________________________________________
42AliAnalysisAlien::AliAnalysisAlien()
43 :AliAnalysisGrid(),
44 fGridJDL(NULL),
45 fPrice(0),
46 fTTL(0),
47 fSplitMaxInputFileNumber(0),
48 fMaxInitFailed(0),
49 fMasterResubmitThreshold(0),
bb885a9e 50 fNtestFiles(0),
c57f56b7 51 fRunNumbers(),
52 fExecutable(),
53 fArguments(),
54 fAnalysisMacro(),
55 fAnalysisSource(),
56 fAdditionalLibs(),
57 fSplitMode(),
58 fAPIVersion(),
59 fROOTVersion(),
60 fAliROOTVersion(),
61 fUser(),
62 fGridWorkingDir(),
63 fGridDataDir(),
64 fDataPattern(),
65 fGridOutputDir(),
66 fOutputArchive(),
67 fOutputFiles(),
68 fInputFormat(),
e7c71df0 69 fDatasetName(),
c57f56b7 70 fJDLName(),
bb885a9e 71 fMergeExcludes(),
f965131e 72 fIncludePath(),
bb885a9e 73 fCloseSE(),
4e5c5506 74 fInputFiles(0),
75 fPackages(0)
c57f56b7 76{
77// Dummy ctor.
78 SetDefaults();
79}
80
81//______________________________________________________________________________
82AliAnalysisAlien::AliAnalysisAlien(const char *name)
83 :AliAnalysisGrid(name),
84 fGridJDL(NULL),
85 fPrice(0),
86 fTTL(0),
87 fSplitMaxInputFileNumber(0),
88 fMaxInitFailed(0),
89 fMasterResubmitThreshold(0),
bb885a9e 90 fNtestFiles(0),
c57f56b7 91 fRunNumbers(),
92 fExecutable(),
93 fArguments(),
94 fAnalysisMacro(),
95 fAnalysisSource(),
96 fAdditionalLibs(),
97 fSplitMode(),
98 fAPIVersion(),
99 fROOTVersion(),
100 fAliROOTVersion(),
101 fUser(),
102 fGridWorkingDir(),
103 fGridDataDir(),
104 fDataPattern(),
105 fGridOutputDir(),
106 fOutputArchive(),
107 fOutputFiles(),
108 fInputFormat(),
e7c71df0 109 fDatasetName(),
c57f56b7 110 fJDLName(),
bb885a9e 111 fMergeExcludes(),
f965131e 112 fIncludePath(),
bb885a9e 113 fCloseSE(),
4e5c5506 114 fInputFiles(0),
115 fPackages(0)
c57f56b7 116{
117// Default ctor.
118 SetDefaults();
119}
120
121//______________________________________________________________________________
122AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
123 :AliAnalysisGrid(other),
124 fGridJDL(NULL),
125 fPrice(other.fPrice),
126 fTTL(other.fTTL),
127 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
128 fMaxInitFailed(other.fMaxInitFailed),
129 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 130 fNtestFiles(other.fNtestFiles),
c57f56b7 131 fRunNumbers(other.fRunNumbers),
132 fExecutable(other.fExecutable),
133 fArguments(other.fArguments),
134 fAnalysisMacro(other.fAnalysisMacro),
135 fAnalysisSource(other.fAnalysisSource),
136 fAdditionalLibs(other.fAdditionalLibs),
137 fSplitMode(other.fSplitMode),
138 fAPIVersion(other.fAPIVersion),
139 fROOTVersion(other.fROOTVersion),
140 fAliROOTVersion(other.fAliROOTVersion),
141 fUser(other.fUser),
142 fGridWorkingDir(other.fGridWorkingDir),
143 fGridDataDir(other.fGridDataDir),
144 fDataPattern(other.fDataPattern),
145 fGridOutputDir(other.fGridOutputDir),
146 fOutputArchive(other.fOutputArchive),
147 fOutputFiles(other.fOutputFiles),
148 fInputFormat(other.fInputFormat),
e7c71df0 149 fDatasetName(other.fDatasetName),
c57f56b7 150 fJDLName(other.fJDLName),
bb885a9e 151 fMergeExcludes(other.fMergeExcludes),
f965131e 152 fIncludePath(other.fIncludePath),
bb885a9e 153 fCloseSE(other.fCloseSE),
4e5c5506 154 fInputFiles(0),
155 fPackages(0)
c57f56b7 156{
157// Copy ctor.
158 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
159 if (other.fInputFiles) {
160 fInputFiles = new TObjArray();
161 TIter next(other.fInputFiles);
162 TObject *obj;
163 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
164 fInputFiles->SetOwner();
165 }
4e5c5506 166 if (other.fPackages) {
167 fPackages = new TObjArray();
168 TIter next(other.fPackages);
169 TObject *obj;
170 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
171 fPackages->SetOwner();
172 }
c57f56b7 173}
174
175//______________________________________________________________________________
176AliAnalysisAlien::~AliAnalysisAlien()
177{
178// Destructor.
179 if (fGridJDL) delete fGridJDL;
180 if (fInputFiles) delete fInputFiles;
4e5c5506 181 if (fPackages) delete fPackages;
c57f56b7 182}
183
184//______________________________________________________________________________
185AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
186{
187// Assignment.
188 if (this != &other) {
189 AliAnalysisGrid::operator=(other);
190 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
191 fPrice = other.fPrice;
192 fTTL = other.fTTL;
193 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
194 fMaxInitFailed = other.fMaxInitFailed;
195 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 196 fNtestFiles = other.fNtestFiles;
c57f56b7 197 fRunNumbers = other.fRunNumbers;
198 fExecutable = other.fExecutable;
199 fArguments = other.fArguments;
200 fAnalysisMacro = other.fAnalysisMacro;
201 fAnalysisSource = other.fAnalysisSource;
202 fAdditionalLibs = other.fAdditionalLibs;
203 fSplitMode = other.fSplitMode;
204 fAPIVersion = other.fAPIVersion;
205 fROOTVersion = other.fROOTVersion;
206 fAliROOTVersion = other.fAliROOTVersion;
207 fUser = other.fUser;
208 fGridWorkingDir = other.fGridWorkingDir;
209 fGridDataDir = other.fGridDataDir;
210 fDataPattern = other.fDataPattern;
211 fGridOutputDir = other.fGridOutputDir;
212 fOutputArchive = other.fOutputArchive;
213 fOutputFiles = other.fOutputFiles;
214 fInputFormat = other.fInputFormat;
e7c71df0 215 fDatasetName = other.fDatasetName;
c57f56b7 216 fJDLName = other.fJDLName;
bb885a9e 217 fMergeExcludes = other.fMergeExcludes;
f965131e 218 fIncludePath = other.fIncludePath;
bb885a9e 219 fCloseSE = other.fCloseSE;
c57f56b7 220 if (other.fInputFiles) {
221 fInputFiles = new TObjArray();
222 TIter next(other.fInputFiles);
223 TObject *obj;
224 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
225 fInputFiles->SetOwner();
226 }
4e5c5506 227 if (other.fPackages) {
228 fPackages = new TObjArray();
229 TIter next(other.fPackages);
230 TObject *obj;
231 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
232 fPackages->SetOwner();
233 }
c57f56b7 234 }
235 return *this;
236}
237
f965131e 238//______________________________________________________________________________
239void AliAnalysisAlien::AddIncludePath(const char *path)
240{
241// Add include path in the remote analysis macro.
242 TString p(path);
243 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
244 else fIncludePath += Form("-I%s ", path);
245}
246
c57f56b7 247//______________________________________________________________________________
248void AliAnalysisAlien::AddRunNumber(Int_t run)
249{
250// Add a run number to the list of runs to be processed.
251 if (fRunNumbers.Length()) fRunNumbers += " ";
252 fRunNumbers += Form("%d", run);
253}
254
255//______________________________________________________________________________
256void AliAnalysisAlien::AddDataFile(const char *lfn)
257{
258// Adds a data file to the input to be analysed. The file should be a valid LFN
259// or point to an existing file in the alien workdir.
260 if (!fInputFiles) fInputFiles = new TObjArray();
261 fInputFiles->Add(new TObjString(lfn));
262}
263
264//______________________________________________________________________________
265Bool_t AliAnalysisAlien::Connect()
266{
267// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
268 if (gGrid && gGrid->IsConnected()) return kTRUE;
269 if (!gSystem->Getenv("alien_API_USER")) {
270 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
271 gSystem->Getenv("UID"));
272 return kFALSE;
273 }
274 if (!gGrid) {
275 Info("Connect", "Trying to connect to AliEn ...");
276 TGrid::Connect("alien://");
277 }
278 if (!gGrid || !gGrid->IsConnected()) {
279 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
280 return kFALSE;
281 }
282 fUser = gGrid->GetUser();
283 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
284 return kTRUE;
285}
286
287//______________________________________________________________________________
288void AliAnalysisAlien::CdWork()
289{
290// Check validity of alien workspace. Create directory if possible.
291 if (!Connect()) {
292 Error("CdWork", "Alien connection required");
293 return;
294 }
295 TString homedir = gGrid->GetHomeDirectory();
296 TString workdir = homedir + fGridWorkingDir;
297 if (!gGrid->Cd(workdir)) {
298 gGrid->Cd(homedir);
299 if (gGrid->Mkdir(workdir)) {
300 gGrid->Cd(fGridWorkingDir);
301 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
302 } else {
303 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
304 workdir.Data(), homedir.Data());
305 fGridWorkingDir = "";
306 }
307 }
308}
309
310//______________________________________________________________________________
311Bool_t AliAnalysisAlien::CheckInputData()
312{
313// Check validity of input data. If necessary, create xml files.
314 if (!fInputFiles && !fRunNumbers.Length()) {
315 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
316 return kFALSE;
317 }
318 // Process declared files
319 Bool_t is_collection = kFALSE;
320 Bool_t is_xml = kFALSE;
321 Bool_t use_tags = kFALSE;
322 Bool_t checked = kFALSE;
323 CdWork();
324 TString file;
325 TString workdir = gGrid->GetHomeDirectory();
326 workdir += fGridWorkingDir;
327 if (fInputFiles) {
328 TObjString *objstr;
329 TIter next(fInputFiles);
330 while ((objstr=(TObjString*)next())) {
331 file = workdir;
332 file += "/";
333 file += objstr->GetString();
334 // Store full lfn path
335 if (FileExists(file)) objstr->SetString(file);
336 else {
337 file = objstr->GetName();
338 if (!FileExists(objstr->GetName())) {
339 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
340 objstr->GetName(), workdir.Data());
341 return kFALSE;
342 }
343 }
344 Bool_t iscoll, isxml, usetags;
345 CheckDataType(file, iscoll, isxml, usetags);
346 if (!checked) {
347 checked = kTRUE;
348 is_collection = iscoll;
349 is_xml = isxml;
350 use_tags = usetags;
351 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
352 } else {
353 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
354 Error("CheckInputData", "Some conflict was found in the types of inputs");
355 return kFALSE;
356 }
357 }
358 }
359 }
360 // Process requested run numbers
361 if (!fRunNumbers.Length()) return kTRUE;
362 // Check validity of alien data directory
363 if (!fGridDataDir.Length()) {
364 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
365 return kFALSE;
366 }
367 if (!gGrid->Cd(fGridDataDir)) {
368 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
369 return kFALSE;
370 }
371 if (is_collection) {
372 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
373 return kFALSE;
374 }
375
376 if (checked && !is_xml) {
377 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
378 return kFALSE;
379 }
380 // Check validity of run number(s)
381 TObjArray *arr;
382 TObjString *os;
383 TString path;
384 if (!checked) {
385 checked = kTRUE;
386 use_tags = fDataPattern.Contains("tag");
387 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
388 }
389 if (use_tags != fDataPattern.Contains("tag")) {
390 Error("CheckInputData", "Cannot mix input files using/not using tags");
391 return kFALSE;
392 }
393 if (fRunNumbers.Length()) {
394 arr = fRunNumbers.Tokenize(" ");
395 TIter next(arr);
396 while ((os=(TObjString*)next())) {
397 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
398 if (!gGrid->Cd(path)) {
399 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
400 return kFALSE;
401 }
402 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
403 TString msg = "\n##### file: ";
404 msg += path;
405 msg += " type: xml_collection;";
406 if (use_tags) msg += " using_tags: Yes";
407 else msg += " using_tags: No";
408 Info("CheckDataType", msg.Data());
409 AddDataFile(path);
410 }
411 delete arr;
412 }
413 return kTRUE;
414}
415
416//______________________________________________________________________________
417Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
418{
419// Create dataset for the grid data directory + run number.
420 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
421 if (!Connect()) {
422 Error("CreateDataset", "Cannot create dataset with no grid connection");
423 return kFALSE;
424 }
425
426 // Cd workspace
427 CdWork();
428 TString workdir = gGrid->GetHomeDirectory();
429 workdir += fGridWorkingDir;
430
431 // Compose the 'find' command arguments
432 TString command;
433 TString options = "-x collection ";
bb885a9e 434 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 435 TString conditions = "";
436
437 TString file;
438 TString path;
439 if (!fRunNumbers.Length()) return kTRUE;
440 // Several runs
441 TObjArray *arr = fRunNumbers.Tokenize(" ");
442 TObjString *os;
443 TIter next(arr);
444 while ((os=(TObjString*)next())) {
445 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
1535db97 446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
447 else file = Form("%s.xml", os->GetString().Data());
448 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
c57f56b7 449 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
450 gGrid->Rm(file);
451 }
452 command = "find ";
453 command += options;
454 command += path;
455 command += pattern;
1535db97 456// conditions = Form(" > %s", file.Data());
c57f56b7 457 command += conditions;
458 TGridResult *res = gGrid->Command(command);
459 if (res) delete res;
1535db97 460 // Write standard output to file
461 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
462 if (TestBit(AliAnalysisGrid::kTest)) break;
463 // Copy xml file to alien space
464 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
c57f56b7 465 if (!FileExists(file)) {
466 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
467 delete arr;
468 return kFALSE;
469 }
c57f56b7 470 }
471 delete arr;
c57f56b7 472 return kTRUE;
473}
474
475//______________________________________________________________________________
476Bool_t AliAnalysisAlien::CreateJDL()
477{
478// Generate a JDL file according to current settings. The name of the file is
479// specified by fJDLName.
480 Bool_t error = kFALSE;
481 TObjArray *arr = 0;
482 Bool_t copy = kTRUE;
483 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
484 Bool_t generate = kTRUE;
485 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
486 if (!Connect()) {
487 Error("CreateJDL", "Alien connection required");
488 return kFALSE;
489 }
490 // Check validity of alien workspace
491 CdWork();
492 TString workdir = gGrid->GetHomeDirectory();
493 workdir += fGridWorkingDir;
494 if (generate) {
495 TObjString *os;
496 if (!fInputFiles) {
497 Error("CreateJDL()", "Define some input files for your analysis.");
498 error = kTRUE;
499 }
500 // Compose list of input files
501 // Check if output files were defined
502 if (!fOutputFiles.Length()) {
503 Error("CreateJDL", "You must define at least one output file");
504 error = kTRUE;
505 }
506 // Check if an output directory was defined and valid
507 if (!fGridOutputDir.Length()) {
508 Error("CreateJDL", "You must define AliEn output directory");
509 error = kTRUE;
510 } else {
511 if (!gGrid->Cd(fGridOutputDir)) {
512 if (gGrid->Mkdir(fGridOutputDir)) {
513 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
514 } else {
515 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
516 error = kTRUE;
517 }
518 }
519 gGrid->Cd(workdir);
520 }
521 // Exit if any error up to now
522 if (error) return kFALSE;
523 // Set JDL fields
524 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
525 fGridJDL->SetExecutable(fExecutable);
526// fGridJDL->SetTTL((UInt_t)fTTL);
527 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
528 if (fMaxInitFailed > 0)
529 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
530 if (fSplitMaxInputFileNumber > 0)
531 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
532 if (fSplitMode.Length())
533 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
534// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
535 if (fAliROOTVersion.Length())
536 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
537 if (fROOTVersion.Length())
538 fGridJDL->AddToPackages("ROOT", fROOTVersion);
539 if (fAPIVersion.Length())
540 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
541 fGridJDL->SetInputDataListFormat(fInputFormat);
542 fGridJDL->SetInputDataList("wn.xml");
543 if (fInputFiles) {
544 TIter next(fInputFiles);
545 while ((os=(TObjString*)next()))
546 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
547 }
548 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
549 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
550 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
551 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
552 if (fAdditionalLibs.Length()) {
553 arr = fAdditionalLibs.Tokenize(" ");
554 TIter next(arr);
555 while ((os=(TObjString*)next())) {
556 if (os->GetString().Contains(".so")) continue;
557 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
558 }
559 delete arr;
560 }
4e5c5506 561 if (fPackages) {
562 TIter next(fPackages);
563 TObject *obj;
564 while ((obj=next()))
565 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
566 }
c57f56b7 567 if (fOutputArchive.Length()) {
568 arr = fOutputArchive.Tokenize(" ");
569 TIter next(arr);
570 while ((os=(TObjString*)next()))
43da816a 571 if (!os->GetString().Contains("@") && fCloseSE.Length())
572 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
573 else
574 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 575 delete arr;
576 }
577 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
578 arr = fOutputFiles.Tokenize(" ");
579 TIter next(arr);
43da816a 580 while ((os=(TObjString*)next())) {
581 if (!os->GetString().Contains("@") && fCloseSE.Length())
582 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
583 else
584 fGridJDL->AddToOutputSandbox(os->GetString());
585 }
c57f56b7 586 delete arr;
587// fGridJDL->SetPrice((UInt_t)fPrice);
588 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
589 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
590 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
591 // Generate the JDL as a string
592 TString sjdl = fGridJDL->Generate();
593 Int_t index;
594 index = sjdl.Index("Executable");
595 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
596 index = sjdl.Index("Split ");
597 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
598 index = sjdl.Index("SplitMaxInputFileNumber");
599 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
600 index = sjdl.Index("InputDataCollection");
601 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
602 index = sjdl.Index("InputFile");
603 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
604 index = sjdl.Index("InputDataList ");
605 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
606 index = sjdl.Index("InputDataListFormat");
607 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
608 index = sjdl.Index("Price");
609 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
610 index = sjdl.Index("Requirements");
611 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
612 index = sjdl.Index("Packages");
613 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
614 index = sjdl.Index("User");
615 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
616 index = sjdl.Index("TTL");
617 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
618 index = sjdl.Index("OutputFile");
619 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
620 index = sjdl.Index("OutputDir");
621 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
622 index = sjdl.Index("OutputArchive");
623 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
624 index = sjdl.Index("MaxInitFailed");
625 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
626 index = sjdl.Index("MasterResubmitThreshold");
627 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
628 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
629 index = sjdl.Index("Validationcommand");
630 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
631 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
632 sjdl.ReplaceAll("(member", "\n (member");
633 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
634 sjdl.ReplaceAll("{", "{\n ");
635 sjdl.ReplaceAll("};", "\n};");
636 sjdl.ReplaceAll("{\n \n", "{\n");
637 sjdl.ReplaceAll("\n\n", "\n");
638 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
639 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
640 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
641 index = sjdl.Index("JDLVariables");
642 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
643 // Write jdl to file
644 ofstream out;
645 out.open(fJDLName.Data(), ios::out);
646 if (out.bad()) {
647 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
648 return kFALSE;
649 }
650 out << sjdl << endl;
651 }
652 // Copy jdl to grid workspace
653 if (!copy) {
654 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
655 } else {
656 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
657 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
658 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
659 if (fAdditionalLibs.Length()) {
e7c71df0 660 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 661 TObjString *os;
662 TIter next(arr);
663 while ((os=(TObjString*)next())) {
c57f56b7 664 if (os->GetString().Contains(".so")) continue;
4e5c5506 665 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 666 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
667 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
668 }
669 delete arr;
670 }
4e5c5506 671 if (fPackages) {
672 TIter next(fPackages);
673 TObject *obj;
674 while ((obj=next())) {
675 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
676 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
677 }
678 }
c57f56b7 679 }
680 return kTRUE;
681}
682
683//______________________________________________________________________________
684Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
685{
686// Returns true if file exists.
687 if (!gGrid) {
688 Error("FileExists", "No connection to grid");
689 return kFALSE;
690 }
691 TGridResult *res = gGrid->Ls(lfn);
692 if (!res) return kFALSE;
693 TMap *map = dynamic_cast<TMap*>(res->At(0));
694 if (!map) {
695 delete res;
696 return kFALSE;
697 }
698 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
699 if (!objs || !objs->GetString().Length()) {
700 delete res;
701 return kFALSE;
702 }
703 delete res;
704 return kTRUE;
705}
706
707//______________________________________________________________________________
708void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
709{
710// Check input data type.
711 is_collection = kFALSE;
712 is_xml = kFALSE;
713 use_tags = kFALSE;
714 if (!gGrid) {
715 Error("CheckDataType", "No connection to grid");
716 return;
717 }
718 is_collection = IsCollection(lfn);
719 TString msg = "\n##### file: ";
720 msg += lfn;
721 if (is_collection) {
722 msg += " type: raw_collection;";
723 // special treatment for collections
724 is_xml = kFALSE;
725 // check for tag files in the collection
726 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
727 if (!res) {
728 msg += " using_tags: No (unknown)";
729 Info("CheckDataType", msg.Data());
730 return;
731 }
732 const char* typeStr = res->GetKey(0, "origLFN");
733 if (!typeStr || !strlen(typeStr)) {
734 msg += " using_tags: No (unknown)";
735 Info("CheckDataType", msg.Data());
736 return;
737 }
738 TString file = typeStr;
739 use_tags = file.Contains(".tag");
740 if (use_tags) msg += " using_tags: Yes";
741 else msg += " using_tags: No";
742 Info("CheckDataType", msg.Data());
743 return;
744 }
745 TString slfn(lfn);
746 slfn.ToLower();
747 is_xml = slfn.Contains(".xml");
748 if (is_xml) {
749 // Open xml collection and check if there are tag files inside
750 msg += " type: xml_collection;";
751 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
752 if (!coll) {
753 msg += " using_tags: No (unknown)";
754 Info("CheckDataType", msg.Data());
755 return;
756 }
757 TMap *map = coll->Next();
758 if (!map) {
759 msg += " using_tags: No (unknown)";
760 Info("CheckDataType", msg.Data());
761 return;
762 }
763 map = (TMap*)map->GetValue("");
764 TString file;
765 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
766 use_tags = file.Contains(".tag");
767 delete coll;
768 if (use_tags) msg += " using_tags: Yes";
769 else msg += " using_tags: No";
770 Info("CheckDataType", msg.Data());
771 return;
772 }
773 use_tags = slfn.Contains(".tag");
774 if (slfn.Contains(".root")) msg += " type: root file;";
775 else msg += " type: unhnown file;";
776 if (use_tags) msg += " using_tags: Yes";
777 else msg += " using_tags: No";
778 Info("CheckDataType", msg.Data());
779}
780
4e5c5506 781//______________________________________________________________________________
782void AliAnalysisAlien::EnablePackage(const char *package)
783{
784// Enables a par file supposed to exist in the current directory.
785 TString pkg(package);
786 pkg.ReplaceAll(".par", "");
787 pkg += ".par";
788 if (gSystem->AccessPathName(pkg)) {
789 Error("EnablePackage", "Package %s not found", pkg.Data());
790 return;
791 }
792 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
793 Info("EnablePackage", "AliEn plugin will use .par packages");
794 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
795 if (!fPackages) {
796 fPackages = new TObjArray();
797 fPackages->SetOwner();
798 }
799 fPackages->Add(new TObjString(pkg));
800}
801
c57f56b7 802//______________________________________________________________________________
803Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
804{
805// Returns true if file is a collection. Functionality duplicated from
806// TAlien::Type() because we don't want to directly depend on TAlien.
807 if (!gGrid) {
808 Error("IsCollection", "No connection to grid");
809 return kFALSE;
810 }
811 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
812 if (!res) return kFALSE;
813 const char* typeStr = res->GetKey(0, "type");
814 if (!typeStr || !strlen(typeStr)) return kFALSE;
815 if (!strcmp(typeStr, "collection")) return kTRUE;
816 delete res;
817 return kFALSE;
818}
819
820//______________________________________________________________________________
821void AliAnalysisAlien::SetDefaults()
822{
823// Set default values for everything. What cannot be filled will be left empty.
824 if (fGridJDL) delete fGridJDL;
825 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
826 fPrice = 1;
827 fTTL = 30000;
828 fSplitMaxInputFileNumber = 100;
829 fMaxInitFailed = 0;
830 fMasterResubmitThreshold = 0;
bb885a9e 831 fNtestFiles = 10;
c57f56b7 832 fRunNumbers = "";
833 fExecutable = "analysis.sh";
834 fArguments = "";
835 fAnalysisMacro = "myAnalysis.C";
836 fAnalysisSource = "";
837 fAdditionalLibs = "";
838 fSplitMode = "se";
839 fAPIVersion = "";
840 fROOTVersion = "";
841 fAliROOTVersion = "";
842 fUser = ""; // Your alien user name
843 fGridWorkingDir = "";
844 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
845 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
846 fGridOutputDir = "output";
847 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
848 fOutputFiles = ""; // Like "AliAODs.root histos.root"
849 fInputFormat = "xml-single";
850 fJDLName = "analysis.jdl";
bb885a9e 851 fMergeExcludes = "";
c57f56b7 852}
853
854//______________________________________________________________________________
855Bool_t AliAnalysisAlien::MergeOutputs()
856{
857// Merge analysis outputs existing in the AliEn space.
858 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
859 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
860 if (!Connect()) {
861 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
862 return kFALSE;
863 }
864 // Get the output path
865 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
866 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
867 if (!gGrid->Cd(output)) {
868 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
869 return kFALSE;
870 }
871 if (!fOutputFiles.Length()) {
872 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
873 return kFALSE;
874 }
875 TObjArray *list = fOutputFiles.Tokenize(" ");
876 TIter next(list);
877 TObjString *str;
878 TString command;
879 TString output_file;
880 Bool_t merged = kTRUE;
881 while((str=(TObjString*)next())) {
882 output_file = str->GetString();
883 Int_t index = output_file.Index("@");
884 if (index > 0) output_file.Remove(index);
bb885a9e 885 if (fMergeExcludes.Length() &&
886 fMergeExcludes.Contains(output_file.Data())) continue;
c57f56b7 887 command = Form("find %s/ *%s", output.Data(), output_file.Data());
888 printf("command: %s\n", command.Data());
889 TGridResult *res = gGrid->Command(command);
890 if (!res) continue;
891 TFileMerger *fm = 0;
892 TIter nextmap(res);
893 TMap *map;
894 while ((map=(TMap*)nextmap())) {
895 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
896 if (!objs || !objs->GetString().Length()) {
897 delete res;
898 continue;
899 }
900 if (!fm) {
901 fm = new TFileMerger(kFALSE);
902 fm->SetFastMethod(kTRUE);
903 fm->OutputFile(output_file);
904 }
905 fm->AddFile(objs->GetString());
906 }
907 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
908 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
909 merged = kFALSE;
910 delete res;
911 continue;
912 }
913 if (!fm->Merge()) {
914 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
915 merged = kFALSE;
916 } else {
917 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
918 }
919 delete fm;
920 delete res;
921 }
922 if (!merged) {
923 Error("MergeOutputs", "Terminate() will NOT be executed");
924 }
925 return merged;
926}
927
bb885a9e 928//______________________________________________________________________________
929void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
930{
931// Use the output files connected to output containers from the analysis manager
932// rather than the files defined by SetOutputFiles
933 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
934 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
935 analysis manager");
936 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
937}
938
c57f56b7 939//______________________________________________________________________________
940void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
941{
942// Start remote grid analysis.
943
43da816a 944 // Check if output files have to be taken from the analysis manager
945 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
946 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
947 if (!mgr || !mgr->IsInitialized()) {
948 Error("StartAnalysis", "You need an initialized analysis manager for this");
949 return;
950 }
951 fOutputFiles = "";
952 TIter next(mgr->GetOutputs());
953 AliAnalysisDataContainer *output;
954 while ((output=(AliAnalysisDataContainer*)next())) {
955 const char *filename = output->GetFileName();
956 if (!(strcmp(filename, "default"))) {
957 if (!mgr->GetOutputEventHandler()) continue;
958 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
959 }
960 if (fOutputFiles.Length()) fOutputFiles += " ";
961 fOutputFiles += filename;
962 }
963 }
f7b1cbc2 964// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 965 if (TestBit(AliAnalysisGrid::kOffline)) {
966 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
967 \n there nor any job run. You can revise the JDL and analysis \
968 \n macro then run the same in \"submit\" mode.");
969 } else if (TestBit(AliAnalysisGrid::kTest)) {
970 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
971 \n dataset.");
972 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
973 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
974 \n space and job submitted.");
975 } else if (TestBit(AliAnalysisGrid::kMerge)) {
976 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
977 return;
978 } else {
979 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
980 }
981
982 if (!Connect()) {
983 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
984 return;
985 }
986 if (!CheckInputData()) {
987 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
988 return;
989 }
990 CreateDataset(fDataPattern);
991 WriteAnalysisFile();
992 WriteAnalysisMacro();
993 WriteExecutable();
994 WriteValidationScript();
995 if (!CreateJDL()) return;
996 if (TestBit(AliAnalysisGrid::kOffline)) return;
997 if (TestBit(AliAnalysisGrid::kTest)) {
998 // Locally testing the analysis
999 Info("StartAnalysis", "\n_______________________________________________________________________ \
1000 \n Running analysis script in a daughter shell as on a worker node \
1001 \n_______________________________________________________________________");
1002 TObjArray *list = fOutputFiles.Tokenize(" ");
1003 TIter next(list);
1004 TObjString *str;
1005 TString output_file;
1006 while((str=(TObjString*)next())) {
1007 output_file = str->GetString();
1008 Int_t index = output_file.Index("@");
1009 if (index > 0) output_file.Remove(index);
43da816a 1010 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1011 }
1012 delete list;
1013 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1014 gSystem->Exec("bash validate.sh");
1015// gSystem->Exec("cat stdout");
1016 return;
1017 }
1018 // Submit AliEn job
1019 CdWork();
1020 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1021 TString jobID = "";
1022 if (res) {
1023 const char *cjobId = res->GetKey(0,"jobId");
1024 if (!cjobId) {
1025 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1026 return;
1027 } else {
1028 Info("StartAnalysis", "\n_______________________________________________________________________ \
1029 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1030 \n_______________________________________________________________________",
1031 fJDLName.Data(), cjobId);
1032 jobID = cjobId;
1033 }
1034 delete res;
1035 }
1036 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1037 \n You may exit at any time and terminate the job later using the option <terminate> \
1038 \n ##################################################################################", jobID.Data());
bb885a9e 1039 //gGrid->Shell();
1040 gSystem->Exec("aliensh");
c57f56b7 1041}
1042
1043//______________________________________________________________________________
1044void AliAnalysisAlien::WriteAnalysisFile()
1045{
1046// Write current analysis manager into the file analysis.root
1047 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1048 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1049 if (!mgr || !mgr->IsInitialized()) {
1050 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1051 return;
1052 }
1053 // Check analysis type
1054 TObject *handler;
1055 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1056 handler = (TObject*)mgr->GetInputEventHandler();
1057 if (handler) {
1058 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1059 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1060 }
1061 TDirectory *cdir = gDirectory;
1062 TFile *file = TFile::Open("analysis.root", "RECREATE");
1063 if (file) {
1064 mgr->Write();
1065 delete file;
1066 }
1067 if (cdir) cdir->cd();
1068 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1069 }
1070 Bool_t copy = kTRUE;
1071 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1072 if (copy) {
1073 CdWork();
1074 TString workdir = gGrid->GetHomeDirectory();
1075 workdir += fGridWorkingDir;
1076 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1077 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1078 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1079 }
1080}
1081
1082//______________________________________________________________________________
1083void AliAnalysisAlien::WriteAnalysisMacro()
1084{
1085// Write the analysis macro that will steer the analysis in grid mode.
1086 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1087 ofstream out;
1088 out.open(fAnalysisMacro.Data(), ios::out);
1089 if (!out.good()) {
1090 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1091 return;
1092 }
1093 TString func = fAnalysisMacro;
1094 TString type = "ESD";
1095 TString comment = "// Analysis using ";
1096 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1097 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1098 type = "AOD";
1099 comment += "AOD";
1100 }
1101 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1102 else comment += " data";
1103 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1104 func.ReplaceAll(".C", "");
1105 out << "void " << func.Data() << "()" << endl;
1106 out << "{" << endl;
1107 out << comment.Data() << endl;
1108 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1109 out << "// load base root libraries" << endl;
1110 out << " gSystem->Load(\"libTree\");" << endl;
1111 out << " gSystem->Load(\"libGeom\");" << endl;
1112 out << " gSystem->Load(\"libVMC\");" << endl;
1113 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1114 out << "// Load analysis framework libraries" << endl;
4e5c5506 1115 if (!fPackages) {
4e5c5506 1116 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1117 out << " gSystem->Load(\"libESD\");" << endl;
1118 out << " gSystem->Load(\"libAOD\");" << endl;
1119 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1120 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1121 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1122 } else {
4e5c5506 1123 TIter next(fPackages);
1124 TObject *obj;
57377eb5 1125 TString pkgname;
1126 Bool_t hasSTEERBase = kFALSE;
1127 Bool_t hasESD = kFALSE;
1128 Bool_t hasAOD = kFALSE;
1129 Bool_t hasANALYSIS = kFALSE;
1130 Bool_t hasANALYSISalice = kFALSE;
1131 Bool_t hasCORRFW = kFALSE;
1132 while ((obj=next())) {
1133 pkgname = obj->GetName();
1134 if (pkgname.Contains("STEERBase")) hasSTEERBase = kTRUE;
1135 if (pkgname.Contains("ESD")) hasESD = kTRUE;
1136 if (pkgname.Contains("AOD")) hasAOD = kTRUE;
1137 if (pkgname.Contains("ANALYSIS") && !pkgname.Contains("ANALYSISalice")) hasANALYSIS = kTRUE;
1138 if (pkgname.Contains("ANALYSISalice")) hasANALYSISalice = kTRUE;
1139 if (pkgname.Contains("CORRFW")) hasCORRFW = kTRUE;
1140 }
1141 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1142 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1143 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1144 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1145 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1146 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1147 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1148 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1149 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1150 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1151 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1152 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1153 out << "// Compile other par packages" << endl;
1154 next.Reset();
4e5c5506 1155 while ((obj=next()))
1156 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1157 }
43da816a 1158 out << "// include path" << endl;
1159 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1160 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1161 if (fAdditionalLibs.Length()) {
1162 out << "// Add aditional AliRoot libraries" << endl;
1163 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1164 TIter next(list);
1165 TObjString *str;
1166 while((str=(TObjString*)next())) {
1167 if (str->GetString().Contains(".so"))
1168 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1169 }
1170 if (list) delete list;
1171 }
1172 out << endl;
c57f56b7 1173 out << "// analysis source to be compiled at runtime (if any)" << endl;
1174 if (fAnalysisSource.Length()) {
1175 TObjArray *list = fAnalysisSource.Tokenize(" ");
1176 TIter next(list);
1177 TObjString *str;
1178 while((str=(TObjString*)next())) {
1179 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1180 }
1181 if (list) delete list;
1182 }
1183 out << endl;
1184 out << "// connect to AliEn and make the chain" << endl;
1185 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1186 if (IsUsingTags()) {
1187 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1188 } else {
1189 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1190 }
1191 out << "// read the analysis manager from file" << endl;
1192 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1193 out << " if (!file) return;" << endl;
1194 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1195 out << " AliAnalysisManager *mgr = 0;" << endl;
1196 out << " TKey *key;" << endl;
1197 out << " while ((key=(TKey*)nextkey())) {" << endl;
1198 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1199 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1200 out << " };" << endl;
1201 out << " if (!mgr) {" << endl;
1202 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1203 out << " return;" << endl;
1204 out << " }" << endl << endl;
1205 out << " mgr->PrintStatus();" << endl;
1206 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1207 out << "}" << endl << endl;
1208 if (IsUsingTags()) {
1209 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1210 out << "{" << endl;
1211 out << "// Create a chain using tags from the xml file." << endl;
1212 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1213 out << " if (!coll) {" << endl;
1214 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1215 out << " return NULL;" << endl;
1216 out << " }" << endl;
1217 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1218 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1219 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1220 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1221 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1222 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1223 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1224 out << " // Check if the cuts configuration file was provided" << endl;
1225 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1226 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1227 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1228 out << " }" << endl;
1229 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1230 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1231 out << " chain->ls();" << endl;
1232 out << " return chain;" << endl;
1233 out << "}" << endl;
1234 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1235 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1236 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1237 msg += " AliLHCTagCuts *lhcCuts,\n";
1238 msg += " AliDetectorTagCuts *detCuts,\n";
1239 msg += " AliEventTagCuts *evCuts)";
1240 Info("WriteAnalysisMacro", msg.Data());
1241 }
1242 } else {
1243 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1244 out << "{" << endl;
1245 out << "// Create a chain using url's from xml file" << endl;
1246 out << " TString treename = type;" << endl;
1247 out << " treename.ToLower();" << endl;
1248 out << " treename += \"Tree\";" << endl;
e02fee64 1249 out << " printf(\"***************************************\\n\");" << endl;
1250 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1251 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1252 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1253 out << " if (!coll) {" << endl;
1254 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1255 out << " return NULL;" << endl;
1256 out << " }" << endl;
1257 out << " TChain *chain = new TChain(treename);" << endl;
1258 out << " coll->Reset();" << endl;
1259 out << " while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
1260 out << " if (!chain->GetNtrees()) {" << endl;
1261 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1262 out << " return NULL;" << endl;
1263 out << " }" << endl;
1264 out << " return chain;" << endl;
1265 out << "}" << endl;
1266 }
4e5c5506 1267 if (fPackages) {
1268 out << "Bool_t SetupPar(const char *package) {" << endl;
1269 out << "// Compile the package and set it up." << endl;
1270 out << " TString pkgdir = package;" << endl;
1271 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
1272 out << " gSystem->Exec(Form(\"tar xvzf %s\", package));" << endl;
1273 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1274 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1275 out << " // Check for BUILD.sh and execute" << endl;
1276 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1277 out << " printf(\"*******************************\\n\");" << endl;
1278 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1279 out << " printf(\"*******************************\\n\");" << endl;
1280 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
1281 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", package);" << endl;
1282 out << " gSystem->ChangeDirectory(cdir);" << endl;
1283 out << " return kFALSE;" << endl;
1284 out << " }" << endl;
1285 out << " } else {" << endl;
1286 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", package);" << endl;
1287 out << " gSystem->ChangeDirectory(cdir);" << endl;
1288 out << " return kFALSE;" << endl;
1289 out << " }" << endl;
1290 out << " // Check for SETUP.C and execute" << endl;
1291 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1292 out << " printf(\"*******************************\\n\");" << endl;
1293 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1294 out << " printf(\"*******************************\\n\");" << endl;
1295 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1296 out << " } else {" << endl;
1297 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", package);" << endl;
1298 out << " gSystem->ChangeDirectory(cdir);" << endl;
1299 out << " return kFALSE;" << endl;
1300 out << " }" << endl;
1301 out << " // Restore original workdir" << endl;
1302 out << " gSystem->ChangeDirectory(cdir);" << endl;
1303 out << " return kTRUE;" << endl;
1304 out << "}" << endl;
1305 }
c57f56b7 1306 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1307 }
1308 Bool_t copy = kTRUE;
1309 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1310 if (copy) {
1311 CdWork();
1312 TString workdir = gGrid->GetHomeDirectory();
1313 workdir += fGridWorkingDir;
1314 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1315 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1316 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1317 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1318 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1319 }
1320 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1321 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1322 }
1323}
1324
1325//______________________________________________________________________________
1326void AliAnalysisAlien::WriteExecutable()
1327{
1328// Generate the alien executable script.
1329 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1330 ofstream out;
1331 out.open(fExecutable.Data(), ios::out);
1332 if (out.bad()) {
1333 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1334 return;
1335 }
1336 out << "#!/bin/bash" << endl;
1337 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1338 out << "echo \"=========================================\"" << endl;
1339 out << "echo \"############## PATH : ##############\"" << endl;
1340 out << "echo $PATH" << endl;
1341 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1342 out << "echo $LD_LIBRARY_PATH" << endl;
1343 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1344 out << "echo $ROOTSYS" << endl;
1345 out << "echo \"############## which root : ##############\"" << endl;
1346 out << "which root" << endl;
1347 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1348 out << "echo $ALICE_ROOT" << endl;
1349 out << "echo \"############## which aliroot : ##############\"" << endl;
1350 out << "which aliroot" << endl;
1351 out << "echo \"=========================================\"" << endl << endl;
1352// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1353 out << "root -b -q ";
1354 out << fAnalysisMacro.Data() << endl << endl;
1355 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1356 }
1357 Bool_t copy = kTRUE;
1358 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1359 if (copy) {
1360 CdWork();
1361 TString workdir = gGrid->GetHomeDirectory();
1362 workdir += fGridWorkingDir;
1363 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1364 if (FileExists(executable)) gGrid->Rm(executable);
1365 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1366 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1367 }
1368}
1369
1370//______________________________________________________________________________
1371void AliAnalysisAlien::WriteValidationScript()
1372{
1373// Generate the alien validation script.
1374 // Generate the validation script
1375 TObjString *os;
1376 if (!Connect()) {
1377 Error("WriteValidationScript", "Alien connection required");
1378 return;
1379 }
1380 TString out_stream = "";
1381 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1382 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1383 ofstream out;
1384 out.open("validate.sh", ios::out);
1385 out << "#!/bin/bash" << endl;
1386 out << "##################################################" << endl;
1387 out << "validateout=`dirname $0`" << endl;
1388 out << "validatetime=`date`" << endl;
1389 out << "validated=\"0\";" << endl;
1390 out << "error=0" << endl;
1391 out << "if [ -z $validateout ]" << endl;
1392 out << "then" << endl;
1393 out << " validateout=\".\"" << endl;
1394 out << "fi" << endl << endl;
1395 out << "cd $validateout;" << endl;
1396 out << "validateworkdir=`pwd`;" << endl << endl;
1397 out << "echo \"*******************************************************\"" << out_stream << endl;
1398 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1399 out << "" << endl;
1400 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1401 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1402 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1403 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1404 out << "ls -la ./" << out_stream << endl;
1405 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1406 out << "##################################################" << endl;
1407 TObjArray *arr = fOutputFiles.Tokenize(" ");
1408 TIter next1(arr);
1409 TString output_file;
1410 while ((os=(TObjString*)next1())) {
1411 output_file = os->GetString();
1412 Int_t index = output_file.Index("@");
1413 if (index > 0) output_file.Remove(index);
1414 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1415 out << " error=1" << endl;
1416 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1417 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1418 out << "fi" << endl;
1419 }
1420 delete arr;
1421 out << "if [ $error = 0 ] ; then" << endl;
1422 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1423 out << "fi" << endl;
1424
1425 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1426 out << "echo \"*******************************************************\"" << out_stream << endl;
1427 out << "cd -" << endl;
1428 out << "exit $error" << endl;
1429 }
1430 Bool_t copy = kTRUE;
1431 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1432 if (copy) {
1433 CdWork();
1434 TString workdir = gGrid->GetHomeDirectory();
1435 workdir += fGridWorkingDir;
1436 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1437 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1438 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1439 }
1440}