]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Adding several checks in order to compile HLT libraries with older AliRoot versions
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
bb885a9e 35#include "AliVEventHandler.h"
36#include "AliAnalysisDataContainer.h"
c57f56b7 37#include "AliAnalysisAlien.h"
38
39ClassImp(AliAnalysisAlien)
40
41//______________________________________________________________________________
42AliAnalysisAlien::AliAnalysisAlien()
43 :AliAnalysisGrid(),
44 fGridJDL(NULL),
45 fPrice(0),
46 fTTL(0),
47 fSplitMaxInputFileNumber(0),
48 fMaxInitFailed(0),
49 fMasterResubmitThreshold(0),
bb885a9e 50 fNtestFiles(0),
c57f56b7 51 fRunNumbers(),
52 fExecutable(),
53 fArguments(),
54 fAnalysisMacro(),
55 fAnalysisSource(),
56 fAdditionalLibs(),
57 fSplitMode(),
58 fAPIVersion(),
59 fROOTVersion(),
60 fAliROOTVersion(),
61 fUser(),
62 fGridWorkingDir(),
63 fGridDataDir(),
64 fDataPattern(),
65 fGridOutputDir(),
66 fOutputArchive(),
67 fOutputFiles(),
68 fInputFormat(),
e7c71df0 69 fDatasetName(),
c57f56b7 70 fJDLName(),
bb885a9e 71 fMergeExcludes(),
f965131e 72 fIncludePath(),
bb885a9e 73 fCloseSE(),
4e5c5506 74 fInputFiles(0),
75 fPackages(0)
c57f56b7 76{
77// Dummy ctor.
78 SetDefaults();
79}
80
81//______________________________________________________________________________
82AliAnalysisAlien::AliAnalysisAlien(const char *name)
83 :AliAnalysisGrid(name),
84 fGridJDL(NULL),
85 fPrice(0),
86 fTTL(0),
87 fSplitMaxInputFileNumber(0),
88 fMaxInitFailed(0),
89 fMasterResubmitThreshold(0),
bb885a9e 90 fNtestFiles(0),
c57f56b7 91 fRunNumbers(),
92 fExecutable(),
93 fArguments(),
94 fAnalysisMacro(),
95 fAnalysisSource(),
96 fAdditionalLibs(),
97 fSplitMode(),
98 fAPIVersion(),
99 fROOTVersion(),
100 fAliROOTVersion(),
101 fUser(),
102 fGridWorkingDir(),
103 fGridDataDir(),
104 fDataPattern(),
105 fGridOutputDir(),
106 fOutputArchive(),
107 fOutputFiles(),
108 fInputFormat(),
e7c71df0 109 fDatasetName(),
c57f56b7 110 fJDLName(),
bb885a9e 111 fMergeExcludes(),
f965131e 112 fIncludePath(),
bb885a9e 113 fCloseSE(),
4e5c5506 114 fInputFiles(0),
115 fPackages(0)
c57f56b7 116{
117// Default ctor.
118 SetDefaults();
119}
120
121//______________________________________________________________________________
122AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
123 :AliAnalysisGrid(other),
124 fGridJDL(NULL),
125 fPrice(other.fPrice),
126 fTTL(other.fTTL),
127 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
128 fMaxInitFailed(other.fMaxInitFailed),
129 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 130 fNtestFiles(other.fNtestFiles),
c57f56b7 131 fRunNumbers(other.fRunNumbers),
132 fExecutable(other.fExecutable),
133 fArguments(other.fArguments),
134 fAnalysisMacro(other.fAnalysisMacro),
135 fAnalysisSource(other.fAnalysisSource),
136 fAdditionalLibs(other.fAdditionalLibs),
137 fSplitMode(other.fSplitMode),
138 fAPIVersion(other.fAPIVersion),
139 fROOTVersion(other.fROOTVersion),
140 fAliROOTVersion(other.fAliROOTVersion),
141 fUser(other.fUser),
142 fGridWorkingDir(other.fGridWorkingDir),
143 fGridDataDir(other.fGridDataDir),
144 fDataPattern(other.fDataPattern),
145 fGridOutputDir(other.fGridOutputDir),
146 fOutputArchive(other.fOutputArchive),
147 fOutputFiles(other.fOutputFiles),
148 fInputFormat(other.fInputFormat),
e7c71df0 149 fDatasetName(other.fDatasetName),
c57f56b7 150 fJDLName(other.fJDLName),
bb885a9e 151 fMergeExcludes(other.fMergeExcludes),
f965131e 152 fIncludePath(other.fIncludePath),
bb885a9e 153 fCloseSE(other.fCloseSE),
4e5c5506 154 fInputFiles(0),
155 fPackages(0)
c57f56b7 156{
157// Copy ctor.
158 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
159 if (other.fInputFiles) {
160 fInputFiles = new TObjArray();
161 TIter next(other.fInputFiles);
162 TObject *obj;
163 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
164 fInputFiles->SetOwner();
165 }
4e5c5506 166 if (other.fPackages) {
167 fPackages = new TObjArray();
168 TIter next(other.fPackages);
169 TObject *obj;
170 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
171 fPackages->SetOwner();
172 }
c57f56b7 173}
174
175//______________________________________________________________________________
176AliAnalysisAlien::~AliAnalysisAlien()
177{
178// Destructor.
179 if (fGridJDL) delete fGridJDL;
180 if (fInputFiles) delete fInputFiles;
4e5c5506 181 if (fPackages) delete fPackages;
c57f56b7 182}
183
184//______________________________________________________________________________
185AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
186{
187// Assignment.
188 if (this != &other) {
189 AliAnalysisGrid::operator=(other);
190 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
191 fPrice = other.fPrice;
192 fTTL = other.fTTL;
193 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
194 fMaxInitFailed = other.fMaxInitFailed;
195 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 196 fNtestFiles = other.fNtestFiles;
c57f56b7 197 fRunNumbers = other.fRunNumbers;
198 fExecutable = other.fExecutable;
199 fArguments = other.fArguments;
200 fAnalysisMacro = other.fAnalysisMacro;
201 fAnalysisSource = other.fAnalysisSource;
202 fAdditionalLibs = other.fAdditionalLibs;
203 fSplitMode = other.fSplitMode;
204 fAPIVersion = other.fAPIVersion;
205 fROOTVersion = other.fROOTVersion;
206 fAliROOTVersion = other.fAliROOTVersion;
207 fUser = other.fUser;
208 fGridWorkingDir = other.fGridWorkingDir;
209 fGridDataDir = other.fGridDataDir;
210 fDataPattern = other.fDataPattern;
211 fGridOutputDir = other.fGridOutputDir;
212 fOutputArchive = other.fOutputArchive;
213 fOutputFiles = other.fOutputFiles;
214 fInputFormat = other.fInputFormat;
e7c71df0 215 fDatasetName = other.fDatasetName;
c57f56b7 216 fJDLName = other.fJDLName;
bb885a9e 217 fMergeExcludes = other.fMergeExcludes;
f965131e 218 fIncludePath = other.fIncludePath;
bb885a9e 219 fCloseSE = other.fCloseSE;
c57f56b7 220 if (other.fInputFiles) {
221 fInputFiles = new TObjArray();
222 TIter next(other.fInputFiles);
223 TObject *obj;
224 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
225 fInputFiles->SetOwner();
226 }
4e5c5506 227 if (other.fPackages) {
228 fPackages = new TObjArray();
229 TIter next(other.fPackages);
230 TObject *obj;
231 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
232 fPackages->SetOwner();
233 }
c57f56b7 234 }
235 return *this;
236}
237
f965131e 238//______________________________________________________________________________
239void AliAnalysisAlien::AddIncludePath(const char *path)
240{
241// Add include path in the remote analysis macro.
242 TString p(path);
243 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
244 else fIncludePath += Form("-I%s ", path);
245}
246
c57f56b7 247//______________________________________________________________________________
248void AliAnalysisAlien::AddRunNumber(Int_t run)
249{
250// Add a run number to the list of runs to be processed.
251 if (fRunNumbers.Length()) fRunNumbers += " ";
252 fRunNumbers += Form("%d", run);
253}
254
255//______________________________________________________________________________
256void AliAnalysisAlien::AddDataFile(const char *lfn)
257{
258// Adds a data file to the input to be analysed. The file should be a valid LFN
259// or point to an existing file in the alien workdir.
260 if (!fInputFiles) fInputFiles = new TObjArray();
261 fInputFiles->Add(new TObjString(lfn));
262}
263
264//______________________________________________________________________________
265Bool_t AliAnalysisAlien::Connect()
266{
267// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
268 if (gGrid && gGrid->IsConnected()) return kTRUE;
269 if (!gSystem->Getenv("alien_API_USER")) {
270 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
271 gSystem->Getenv("UID"));
272 return kFALSE;
273 }
274 if (!gGrid) {
275 Info("Connect", "Trying to connect to AliEn ...");
276 TGrid::Connect("alien://");
277 }
278 if (!gGrid || !gGrid->IsConnected()) {
279 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
280 return kFALSE;
281 }
282 fUser = gGrid->GetUser();
283 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
284 return kTRUE;
285}
286
287//______________________________________________________________________________
288void AliAnalysisAlien::CdWork()
289{
290// Check validity of alien workspace. Create directory if possible.
291 if (!Connect()) {
292 Error("CdWork", "Alien connection required");
293 return;
294 }
295 TString homedir = gGrid->GetHomeDirectory();
296 TString workdir = homedir + fGridWorkingDir;
297 if (!gGrid->Cd(workdir)) {
298 gGrid->Cd(homedir);
299 if (gGrid->Mkdir(workdir)) {
300 gGrid->Cd(fGridWorkingDir);
301 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
302 } else {
303 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
304 workdir.Data(), homedir.Data());
305 fGridWorkingDir = "";
306 }
307 }
308}
309
310//______________________________________________________________________________
311Bool_t AliAnalysisAlien::CheckInputData()
312{
313// Check validity of input data. If necessary, create xml files.
314 if (!fInputFiles && !fRunNumbers.Length()) {
315 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
316 return kFALSE;
317 }
318 // Process declared files
319 Bool_t is_collection = kFALSE;
320 Bool_t is_xml = kFALSE;
321 Bool_t use_tags = kFALSE;
322 Bool_t checked = kFALSE;
323 CdWork();
324 TString file;
325 TString workdir = gGrid->GetHomeDirectory();
326 workdir += fGridWorkingDir;
327 if (fInputFiles) {
328 TObjString *objstr;
329 TIter next(fInputFiles);
330 while ((objstr=(TObjString*)next())) {
331 file = workdir;
332 file += "/";
333 file += objstr->GetString();
334 // Store full lfn path
335 if (FileExists(file)) objstr->SetString(file);
336 else {
337 file = objstr->GetName();
338 if (!FileExists(objstr->GetName())) {
339 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
340 objstr->GetName(), workdir.Data());
341 return kFALSE;
342 }
343 }
344 Bool_t iscoll, isxml, usetags;
345 CheckDataType(file, iscoll, isxml, usetags);
346 if (!checked) {
347 checked = kTRUE;
348 is_collection = iscoll;
349 is_xml = isxml;
350 use_tags = usetags;
351 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
352 } else {
353 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
354 Error("CheckInputData", "Some conflict was found in the types of inputs");
355 return kFALSE;
356 }
357 }
358 }
359 }
360 // Process requested run numbers
361 if (!fRunNumbers.Length()) return kTRUE;
362 // Check validity of alien data directory
363 if (!fGridDataDir.Length()) {
364 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
365 return kFALSE;
366 }
367 if (!gGrid->Cd(fGridDataDir)) {
368 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
369 return kFALSE;
370 }
371 if (is_collection) {
372 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
373 return kFALSE;
374 }
375
376 if (checked && !is_xml) {
377 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
378 return kFALSE;
379 }
380 // Check validity of run number(s)
381 TObjArray *arr;
382 TObjString *os;
383 TString path;
384 if (!checked) {
385 checked = kTRUE;
386 use_tags = fDataPattern.Contains("tag");
387 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
388 }
389 if (use_tags != fDataPattern.Contains("tag")) {
390 Error("CheckInputData", "Cannot mix input files using/not using tags");
391 return kFALSE;
392 }
393 if (fRunNumbers.Length()) {
394 arr = fRunNumbers.Tokenize(" ");
395 TIter next(arr);
396 while ((os=(TObjString*)next())) {
397 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
398 if (!gGrid->Cd(path)) {
399 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
400 return kFALSE;
401 }
402 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
403 TString msg = "\n##### file: ";
404 msg += path;
405 msg += " type: xml_collection;";
406 if (use_tags) msg += " using_tags: Yes";
407 else msg += " using_tags: No";
408 Info("CheckDataType", msg.Data());
409 AddDataFile(path);
410 }
411 delete arr;
412 }
413 return kTRUE;
414}
415
416//______________________________________________________________________________
417Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
418{
419// Create dataset for the grid data directory + run number.
420 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
421 if (!Connect()) {
422 Error("CreateDataset", "Cannot create dataset with no grid connection");
423 return kFALSE;
424 }
425
426 // Cd workspace
427 CdWork();
428 TString workdir = gGrid->GetHomeDirectory();
429 workdir += fGridWorkingDir;
430
431 // Compose the 'find' command arguments
432 TString command;
433 TString options = "-x collection ";
bb885a9e 434 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 435 TString conditions = "";
436
437 TString file;
438 TString path;
439 if (!fRunNumbers.Length()) return kTRUE;
440 // Several runs
441 TObjArray *arr = fRunNumbers.Tokenize(" ");
442 TObjString *os;
443 TIter next(arr);
444 while ((os=(TObjString*)next())) {
445 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
1535db97 446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
447 else file = Form("%s.xml", os->GetString().Data());
448 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
c57f56b7 449 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
450 gGrid->Rm(file);
451 }
452 command = "find ";
453 command += options;
454 command += path;
455 command += pattern;
1535db97 456// conditions = Form(" > %s", file.Data());
c57f56b7 457 command += conditions;
458 TGridResult *res = gGrid->Command(command);
459 if (res) delete res;
1535db97 460 // Write standard output to file
461 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
462 if (TestBit(AliAnalysisGrid::kTest)) break;
463 // Copy xml file to alien space
464 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
c57f56b7 465 if (!FileExists(file)) {
466 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
467 delete arr;
468 return kFALSE;
469 }
c57f56b7 470 }
471 delete arr;
c57f56b7 472 return kTRUE;
473}
474
475//______________________________________________________________________________
476Bool_t AliAnalysisAlien::CreateJDL()
477{
478// Generate a JDL file according to current settings. The name of the file is
479// specified by fJDLName.
480 Bool_t error = kFALSE;
481 TObjArray *arr = 0;
482 Bool_t copy = kTRUE;
483 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
484 Bool_t generate = kTRUE;
485 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
486 if (!Connect()) {
487 Error("CreateJDL", "Alien connection required");
488 return kFALSE;
489 }
490 // Check validity of alien workspace
491 CdWork();
492 TString workdir = gGrid->GetHomeDirectory();
493 workdir += fGridWorkingDir;
494 if (generate) {
495 TObjString *os;
496 if (!fInputFiles) {
497 Error("CreateJDL()", "Define some input files for your analysis.");
498 error = kTRUE;
499 }
500 // Compose list of input files
501 // Check if output files were defined
502 if (!fOutputFiles.Length()) {
503 Error("CreateJDL", "You must define at least one output file");
504 error = kTRUE;
505 }
506 // Check if an output directory was defined and valid
507 if (!fGridOutputDir.Length()) {
508 Error("CreateJDL", "You must define AliEn output directory");
509 error = kTRUE;
510 } else {
511 if (!gGrid->Cd(fGridOutputDir)) {
512 if (gGrid->Mkdir(fGridOutputDir)) {
513 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
514 } else {
515 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
516 error = kTRUE;
517 }
518 }
519 gGrid->Cd(workdir);
520 }
521 // Exit if any error up to now
522 if (error) return kFALSE;
523 // Set JDL fields
524 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
525 fGridJDL->SetExecutable(fExecutable);
526// fGridJDL->SetTTL((UInt_t)fTTL);
527 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
528 if (fMaxInitFailed > 0)
529 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
530 if (fSplitMaxInputFileNumber > 0)
531 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
532 if (fSplitMode.Length())
533 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
534// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
535 if (fAliROOTVersion.Length())
536 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
537 if (fROOTVersion.Length())
538 fGridJDL->AddToPackages("ROOT", fROOTVersion);
539 if (fAPIVersion.Length())
540 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
541 fGridJDL->SetInputDataListFormat(fInputFormat);
542 fGridJDL->SetInputDataList("wn.xml");
543 if (fInputFiles) {
544 TIter next(fInputFiles);
545 while ((os=(TObjString*)next()))
546 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
547 }
548 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
549 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
550 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
551 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
552 if (fAdditionalLibs.Length()) {
553 arr = fAdditionalLibs.Tokenize(" ");
554 TIter next(arr);
555 while ((os=(TObjString*)next())) {
556 if (os->GetString().Contains(".so")) continue;
557 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
558 }
559 delete arr;
560 }
4e5c5506 561 if (fPackages) {
562 TIter next(fPackages);
563 TObject *obj;
564 while ((obj=next()))
565 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
566 }
c57f56b7 567 if (fOutputArchive.Length()) {
568 arr = fOutputArchive.Tokenize(" ");
569 TIter next(arr);
570 while ((os=(TObjString*)next()))
43da816a 571 if (!os->GetString().Contains("@") && fCloseSE.Length())
572 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
573 else
574 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 575 delete arr;
576 }
577 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
578 arr = fOutputFiles.Tokenize(" ");
579 TIter next(arr);
43da816a 580 while ((os=(TObjString*)next())) {
581 if (!os->GetString().Contains("@") && fCloseSE.Length())
582 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
583 else
584 fGridJDL->AddToOutputSandbox(os->GetString());
585 }
c57f56b7 586 delete arr;
587// fGridJDL->SetPrice((UInt_t)fPrice);
588 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
589 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
590 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
591 // Generate the JDL as a string
592 TString sjdl = fGridJDL->Generate();
593 Int_t index;
594 index = sjdl.Index("Executable");
595 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
596 index = sjdl.Index("Split ");
597 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
598 index = sjdl.Index("SplitMaxInputFileNumber");
599 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
600 index = sjdl.Index("InputDataCollection");
601 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
602 index = sjdl.Index("InputFile");
603 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
604 index = sjdl.Index("InputDataList ");
605 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
606 index = sjdl.Index("InputDataListFormat");
607 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
608 index = sjdl.Index("Price");
609 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
610 index = sjdl.Index("Requirements");
611 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
612 index = sjdl.Index("Packages");
613 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
614 index = sjdl.Index("User");
615 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
616 index = sjdl.Index("TTL");
617 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
618 index = sjdl.Index("OutputFile");
619 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
620 index = sjdl.Index("OutputDir");
621 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
622 index = sjdl.Index("OutputArchive");
623 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
624 index = sjdl.Index("MaxInitFailed");
625 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
626 index = sjdl.Index("MasterResubmitThreshold");
627 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
628 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
629 index = sjdl.Index("Validationcommand");
630 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
631 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
632 sjdl.ReplaceAll("(member", "\n (member");
633 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
634 sjdl.ReplaceAll("{", "{\n ");
635 sjdl.ReplaceAll("};", "\n};");
636 sjdl.ReplaceAll("{\n \n", "{\n");
637 sjdl.ReplaceAll("\n\n", "\n");
638 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
639 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
640 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
641 index = sjdl.Index("JDLVariables");
642 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
643 // Write jdl to file
644 ofstream out;
645 out.open(fJDLName.Data(), ios::out);
646 if (out.bad()) {
647 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
648 return kFALSE;
649 }
650 out << sjdl << endl;
651 }
652 // Copy jdl to grid workspace
653 if (!copy) {
654 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
655 } else {
656 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
657 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
658 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
659 if (fAdditionalLibs.Length()) {
e7c71df0 660 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 661 TObjString *os;
662 TIter next(arr);
663 while ((os=(TObjString*)next())) {
c57f56b7 664 if (os->GetString().Contains(".so")) continue;
4e5c5506 665 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 666 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
667 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
668 }
669 delete arr;
670 }
4e5c5506 671 if (fPackages) {
672 TIter next(fPackages);
673 TObject *obj;
674 while ((obj=next())) {
675 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
676 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
677 }
678 }
c57f56b7 679 }
680 return kTRUE;
681}
682
683//______________________________________________________________________________
684Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
685{
686// Returns true if file exists.
687 if (!gGrid) {
688 Error("FileExists", "No connection to grid");
689 return kFALSE;
690 }
691 TGridResult *res = gGrid->Ls(lfn);
692 if (!res) return kFALSE;
693 TMap *map = dynamic_cast<TMap*>(res->At(0));
694 if (!map) {
695 delete res;
696 return kFALSE;
697 }
698 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
699 if (!objs || !objs->GetString().Length()) {
700 delete res;
701 return kFALSE;
702 }
703 delete res;
704 return kTRUE;
705}
706
707//______________________________________________________________________________
708void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
709{
710// Check input data type.
711 is_collection = kFALSE;
712 is_xml = kFALSE;
713 use_tags = kFALSE;
714 if (!gGrid) {
715 Error("CheckDataType", "No connection to grid");
716 return;
717 }
718 is_collection = IsCollection(lfn);
719 TString msg = "\n##### file: ";
720 msg += lfn;
721 if (is_collection) {
722 msg += " type: raw_collection;";
723 // special treatment for collections
724 is_xml = kFALSE;
725 // check for tag files in the collection
726 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
727 if (!res) {
728 msg += " using_tags: No (unknown)";
729 Info("CheckDataType", msg.Data());
730 return;
731 }
732 const char* typeStr = res->GetKey(0, "origLFN");
733 if (!typeStr || !strlen(typeStr)) {
734 msg += " using_tags: No (unknown)";
735 Info("CheckDataType", msg.Data());
736 return;
737 }
738 TString file = typeStr;
739 use_tags = file.Contains(".tag");
740 if (use_tags) msg += " using_tags: Yes";
741 else msg += " using_tags: No";
742 Info("CheckDataType", msg.Data());
743 return;
744 }
745 TString slfn(lfn);
746 slfn.ToLower();
747 is_xml = slfn.Contains(".xml");
748 if (is_xml) {
749 // Open xml collection and check if there are tag files inside
750 msg += " type: xml_collection;";
751 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
752 if (!coll) {
753 msg += " using_tags: No (unknown)";
754 Info("CheckDataType", msg.Data());
755 return;
756 }
757 TMap *map = coll->Next();
758 if (!map) {
759 msg += " using_tags: No (unknown)";
760 Info("CheckDataType", msg.Data());
761 return;
762 }
763 map = (TMap*)map->GetValue("");
764 TString file;
765 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
766 use_tags = file.Contains(".tag");
767 delete coll;
768 if (use_tags) msg += " using_tags: Yes";
769 else msg += " using_tags: No";
770 Info("CheckDataType", msg.Data());
771 return;
772 }
773 use_tags = slfn.Contains(".tag");
774 if (slfn.Contains(".root")) msg += " type: root file;";
775 else msg += " type: unhnown file;";
776 if (use_tags) msg += " using_tags: Yes";
777 else msg += " using_tags: No";
778 Info("CheckDataType", msg.Data());
779}
780
4e5c5506 781//______________________________________________________________________________
782void AliAnalysisAlien::EnablePackage(const char *package)
783{
784// Enables a par file supposed to exist in the current directory.
785 TString pkg(package);
786 pkg.ReplaceAll(".par", "");
787 pkg += ".par";
788 if (gSystem->AccessPathName(pkg)) {
789 Error("EnablePackage", "Package %s not found", pkg.Data());
790 return;
791 }
792 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
793 Info("EnablePackage", "AliEn plugin will use .par packages");
794 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
795 if (!fPackages) {
796 fPackages = new TObjArray();
797 fPackages->SetOwner();
798 }
799 fPackages->Add(new TObjString(pkg));
800}
801
c57f56b7 802//______________________________________________________________________________
803Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
804{
805// Returns true if file is a collection. Functionality duplicated from
806// TAlien::Type() because we don't want to directly depend on TAlien.
807 if (!gGrid) {
808 Error("IsCollection", "No connection to grid");
809 return kFALSE;
810 }
811 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
812 if (!res) return kFALSE;
813 const char* typeStr = res->GetKey(0, "type");
814 if (!typeStr || !strlen(typeStr)) return kFALSE;
815 if (!strcmp(typeStr, "collection")) return kTRUE;
816 delete res;
817 return kFALSE;
818}
819
820//______________________________________________________________________________
821void AliAnalysisAlien::SetDefaults()
822{
823// Set default values for everything. What cannot be filled will be left empty.
824 if (fGridJDL) delete fGridJDL;
825 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
826 fPrice = 1;
827 fTTL = 30000;
828 fSplitMaxInputFileNumber = 100;
829 fMaxInitFailed = 0;
830 fMasterResubmitThreshold = 0;
bb885a9e 831 fNtestFiles = 10;
c57f56b7 832 fRunNumbers = "";
833 fExecutable = "analysis.sh";
834 fArguments = "";
835 fAnalysisMacro = "myAnalysis.C";
836 fAnalysisSource = "";
837 fAdditionalLibs = "";
838 fSplitMode = "se";
839 fAPIVersion = "";
840 fROOTVersion = "";
841 fAliROOTVersion = "";
842 fUser = ""; // Your alien user name
843 fGridWorkingDir = "";
844 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
845 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
846 fGridOutputDir = "output";
847 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
848 fOutputFiles = ""; // Like "AliAODs.root histos.root"
849 fInputFormat = "xml-single";
850 fJDLName = "analysis.jdl";
bb885a9e 851 fMergeExcludes = "";
c57f56b7 852}
853
854//______________________________________________________________________________
855Bool_t AliAnalysisAlien::MergeOutputs()
856{
857// Merge analysis outputs existing in the AliEn space.
858 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
859 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
860 if (!Connect()) {
861 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
862 return kFALSE;
863 }
864 // Get the output path
865 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
866 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
867 if (!gGrid->Cd(output)) {
868 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
869 return kFALSE;
870 }
871 if (!fOutputFiles.Length()) {
872 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
873 return kFALSE;
874 }
875 TObjArray *list = fOutputFiles.Tokenize(" ");
876 TIter next(list);
877 TObjString *str;
878 TString command;
879 TString output_file;
880 Bool_t merged = kTRUE;
881 while((str=(TObjString*)next())) {
882 output_file = str->GetString();
883 Int_t index = output_file.Index("@");
884 if (index > 0) output_file.Remove(index);
bb885a9e 885 if (fMergeExcludes.Length() &&
886 fMergeExcludes.Contains(output_file.Data())) continue;
c57f56b7 887 command = Form("find %s/ *%s", output.Data(), output_file.Data());
888 printf("command: %s\n", command.Data());
889 TGridResult *res = gGrid->Command(command);
890 if (!res) continue;
891 TFileMerger *fm = 0;
892 TIter nextmap(res);
893 TMap *map;
894 while ((map=(TMap*)nextmap())) {
895 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
896 if (!objs || !objs->GetString().Length()) {
897 delete res;
898 continue;
899 }
900 if (!fm) {
901 fm = new TFileMerger(kFALSE);
902 fm->SetFastMethod(kTRUE);
903 fm->OutputFile(output_file);
904 }
905 fm->AddFile(objs->GetString());
906 }
907 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
908 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
909 merged = kFALSE;
910 delete res;
911 continue;
912 }
913 if (!fm->Merge()) {
914 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
915 merged = kFALSE;
916 } else {
917 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
918 }
919 delete fm;
920 delete res;
921 }
922 if (!merged) {
923 Error("MergeOutputs", "Terminate() will NOT be executed");
924 }
925 return merged;
926}
927
bb885a9e 928//______________________________________________________________________________
929void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
930{
931// Use the output files connected to output containers from the analysis manager
932// rather than the files defined by SetOutputFiles
933 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
934 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
935 analysis manager");
936 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
937}
938
c57f56b7 939//______________________________________________________________________________
940void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
941{
942// Start remote grid analysis.
943
43da816a 944 // Check if output files have to be taken from the analysis manager
945 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
946 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
947 if (!mgr || !mgr->IsInitialized()) {
948 Error("StartAnalysis", "You need an initialized analysis manager for this");
949 return;
950 }
951 fOutputFiles = "";
952 TIter next(mgr->GetOutputs());
953 AliAnalysisDataContainer *output;
954 while ((output=(AliAnalysisDataContainer*)next())) {
955 const char *filename = output->GetFileName();
956 if (!(strcmp(filename, "default"))) {
957 if (!mgr->GetOutputEventHandler()) continue;
958 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
959 }
960 if (fOutputFiles.Length()) fOutputFiles += " ";
961 fOutputFiles += filename;
962 }
c07b9ce2 963 // Add extra files registered to the analysis manager
964 if (mgr->GetExtraFiles().Length()) {
965 if (fOutputFiles.Length()) fOutputFiles += " ";
966 fOutputFiles += mgr->GetExtraFiles();
967 }
43da816a 968 }
f7b1cbc2 969// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 970 if (TestBit(AliAnalysisGrid::kOffline)) {
971 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
972 \n there nor any job run. You can revise the JDL and analysis \
973 \n macro then run the same in \"submit\" mode.");
974 } else if (TestBit(AliAnalysisGrid::kTest)) {
975 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
976 \n dataset.");
977 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
978 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
979 \n space and job submitted.");
980 } else if (TestBit(AliAnalysisGrid::kMerge)) {
981 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
982 return;
983 } else {
984 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
985 }
986
987 if (!Connect()) {
988 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
989 return;
990 }
991 if (!CheckInputData()) {
992 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
993 return;
994 }
995 CreateDataset(fDataPattern);
996 WriteAnalysisFile();
997 WriteAnalysisMacro();
998 WriteExecutable();
999 WriteValidationScript();
1000 if (!CreateJDL()) return;
1001 if (TestBit(AliAnalysisGrid::kOffline)) return;
1002 if (TestBit(AliAnalysisGrid::kTest)) {
1003 // Locally testing the analysis
1004 Info("StartAnalysis", "\n_______________________________________________________________________ \
1005 \n Running analysis script in a daughter shell as on a worker node \
1006 \n_______________________________________________________________________");
1007 TObjArray *list = fOutputFiles.Tokenize(" ");
1008 TIter next(list);
1009 TObjString *str;
1010 TString output_file;
1011 while((str=(TObjString*)next())) {
1012 output_file = str->GetString();
1013 Int_t index = output_file.Index("@");
1014 if (index > 0) output_file.Remove(index);
43da816a 1015 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1016 }
1017 delete list;
1018 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1019 gSystem->Exec("bash validate.sh");
1020// gSystem->Exec("cat stdout");
1021 return;
1022 }
1023 // Submit AliEn job
1024 CdWork();
1025 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1026 TString jobID = "";
1027 if (res) {
1028 const char *cjobId = res->GetKey(0,"jobId");
1029 if (!cjobId) {
1030 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1031 return;
1032 } else {
1033 Info("StartAnalysis", "\n_______________________________________________________________________ \
1034 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1035 \n_______________________________________________________________________",
1036 fJDLName.Data(), cjobId);
1037 jobID = cjobId;
1038 }
1039 delete res;
1040 }
1041 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1042 \n You may exit at any time and terminate the job later using the option <terminate> \
1043 \n ##################################################################################", jobID.Data());
bb885a9e 1044 //gGrid->Shell();
1045 gSystem->Exec("aliensh");
c57f56b7 1046}
1047
1048//______________________________________________________________________________
1049void AliAnalysisAlien::WriteAnalysisFile()
1050{
1051// Write current analysis manager into the file analysis.root
1052 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1053 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1054 if (!mgr || !mgr->IsInitialized()) {
1055 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1056 return;
1057 }
1058 // Check analysis type
1059 TObject *handler;
1060 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1061 handler = (TObject*)mgr->GetInputEventHandler();
1062 if (handler) {
1063 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1064 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1065 }
1066 TDirectory *cdir = gDirectory;
1067 TFile *file = TFile::Open("analysis.root", "RECREATE");
1068 if (file) {
1069 mgr->Write();
1070 delete file;
1071 }
1072 if (cdir) cdir->cd();
1073 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1074 }
1075 Bool_t copy = kTRUE;
1076 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1077 if (copy) {
1078 CdWork();
1079 TString workdir = gGrid->GetHomeDirectory();
1080 workdir += fGridWorkingDir;
1081 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1082 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1083 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1084 }
1085}
1086
1087//______________________________________________________________________________
1088void AliAnalysisAlien::WriteAnalysisMacro()
1089{
1090// Write the analysis macro that will steer the analysis in grid mode.
1091 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1092 ofstream out;
1093 out.open(fAnalysisMacro.Data(), ios::out);
1094 if (!out.good()) {
1095 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1096 return;
1097 }
1098 TString func = fAnalysisMacro;
1099 TString type = "ESD";
1100 TString comment = "// Analysis using ";
1101 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1102 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1103 type = "AOD";
1104 comment += "AOD";
1105 }
1106 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1107 else comment += " data";
1108 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1109 func.ReplaceAll(".C", "");
1110 out << "void " << func.Data() << "()" << endl;
1111 out << "{" << endl;
1112 out << comment.Data() << endl;
1113 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1114 out << "// load base root libraries" << endl;
1115 out << " gSystem->Load(\"libTree\");" << endl;
1116 out << " gSystem->Load(\"libGeom\");" << endl;
1117 out << " gSystem->Load(\"libVMC\");" << endl;
1118 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1119 out << "// Load analysis framework libraries" << endl;
4e5c5506 1120 if (!fPackages) {
4e5c5506 1121 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1122 out << " gSystem->Load(\"libESD\");" << endl;
1123 out << " gSystem->Load(\"libAOD\");" << endl;
1124 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1125 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1126 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1127 } else {
4e5c5506 1128 TIter next(fPackages);
1129 TObject *obj;
57377eb5 1130 TString pkgname;
1131 Bool_t hasSTEERBase = kFALSE;
1132 Bool_t hasESD = kFALSE;
1133 Bool_t hasAOD = kFALSE;
1134 Bool_t hasANALYSIS = kFALSE;
1135 Bool_t hasANALYSISalice = kFALSE;
1136 Bool_t hasCORRFW = kFALSE;
1137 while ((obj=next())) {
1138 pkgname = obj->GetName();
1139 if (pkgname.Contains("STEERBase")) hasSTEERBase = kTRUE;
1140 if (pkgname.Contains("ESD")) hasESD = kTRUE;
1141 if (pkgname.Contains("AOD")) hasAOD = kTRUE;
1142 if (pkgname.Contains("ANALYSIS") && !pkgname.Contains("ANALYSISalice")) hasANALYSIS = kTRUE;
1143 if (pkgname.Contains("ANALYSISalice")) hasANALYSISalice = kTRUE;
1144 if (pkgname.Contains("CORRFW")) hasCORRFW = kTRUE;
1145 }
1146 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1147 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1148 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1149 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1150 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1151 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1152 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1153 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1154 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1155 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1156 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1157 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1158 out << "// Compile other par packages" << endl;
1159 next.Reset();
fcc9bb6f 1160 while ((obj=next())) {
1161 pkgname = obj->GetName();
1162 if (pkgname.Contains("STEERBase") ||
1163 pkgname.Contains("ESD") ||
1164 pkgname.Contains("AOD") ||
1165 pkgname.Contains("ANALYSIS") ||
1166 pkgname.Contains("CORRFW")) continue;
4e5c5506 1167 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1168 }
4e5c5506 1169 }
43da816a 1170 out << "// include path" << endl;
1171 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1172 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1173 if (fAdditionalLibs.Length()) {
1174 out << "// Add aditional AliRoot libraries" << endl;
1175 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1176 TIter next(list);
1177 TObjString *str;
1178 while((str=(TObjString*)next())) {
1179 if (str->GetString().Contains(".so"))
1180 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1181 }
1182 if (list) delete list;
1183 }
1184 out << endl;
c57f56b7 1185 out << "// analysis source to be compiled at runtime (if any)" << endl;
1186 if (fAnalysisSource.Length()) {
1187 TObjArray *list = fAnalysisSource.Tokenize(" ");
1188 TIter next(list);
1189 TObjString *str;
1190 while((str=(TObjString*)next())) {
1191 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1192 }
1193 if (list) delete list;
1194 }
1195 out << endl;
1196 out << "// connect to AliEn and make the chain" << endl;
1197 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1198 if (IsUsingTags()) {
1199 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1200 } else {
1201 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1202 }
1203 out << "// read the analysis manager from file" << endl;
1204 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1205 out << " if (!file) return;" << endl;
1206 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1207 out << " AliAnalysisManager *mgr = 0;" << endl;
1208 out << " TKey *key;" << endl;
1209 out << " while ((key=(TKey*)nextkey())) {" << endl;
1210 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1211 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1212 out << " };" << endl;
1213 out << " if (!mgr) {" << endl;
1214 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1215 out << " return;" << endl;
1216 out << " }" << endl << endl;
1217 out << " mgr->PrintStatus();" << endl;
1218 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1219 out << "}" << endl << endl;
1220 if (IsUsingTags()) {
1221 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1222 out << "{" << endl;
1223 out << "// Create a chain using tags from the xml file." << endl;
1224 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1225 out << " if (!coll) {" << endl;
1226 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1227 out << " return NULL;" << endl;
1228 out << " }" << endl;
1229 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1230 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1231 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1232 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1233 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1234 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1235 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1236 out << " // Check if the cuts configuration file was provided" << endl;
1237 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1238 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1239 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1240 out << " }" << endl;
1241 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1242 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1243 out << " chain->ls();" << endl;
1244 out << " return chain;" << endl;
fcc9bb6f 1245 out << "}" << endl << endl;
c57f56b7 1246 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1247 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1248 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1249 msg += " AliLHCTagCuts *lhcCuts,\n";
1250 msg += " AliDetectorTagCuts *detCuts,\n";
1251 msg += " AliEventTagCuts *evCuts)";
1252 Info("WriteAnalysisMacro", msg.Data());
1253 }
1254 } else {
fcc9bb6f 1255 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1256 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1257 out << "{" << endl;
1258 out << "// Create a chain using url's from xml file" << endl;
1259 out << " TString treename = type;" << endl;
1260 out << " treename.ToLower();" << endl;
1261 out << " treename += \"Tree\";" << endl;
e02fee64 1262 out << " printf(\"***************************************\\n\");" << endl;
1263 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1264 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1265 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1266 out << " if (!coll) {" << endl;
1267 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1268 out << " return NULL;" << endl;
1269 out << " }" << endl;
1270 out << " TChain *chain = new TChain(treename);" << endl;
1271 out << " coll->Reset();" << endl;
1272 out << " while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
1273 out << " if (!chain->GetNtrees()) {" << endl;
1274 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1275 out << " return NULL;" << endl;
1276 out << " }" << endl;
1277 out << " return chain;" << endl;
fcc9bb6f 1278 out << "}" << endl << endl;
c57f56b7 1279 }
4e5c5506 1280 if (fPackages) {
fcc9bb6f 1281 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1282 out << "Bool_t SetupPar(const char *package) {" << endl;
1283 out << "// Compile the package and set it up." << endl;
1284 out << " TString pkgdir = package;" << endl;
1285 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1286 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1287 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1288 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1289 out << " // Check for BUILD.sh and execute" << endl;
1290 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1291 out << " printf(\"*******************************\\n\");" << endl;
1292 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1293 out << " printf(\"*******************************\\n\");" << endl;
1294 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1295 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1296 out << " gSystem->ChangeDirectory(cdir);" << endl;
1297 out << " return kFALSE;" << endl;
1298 out << " }" << endl;
1299 out << " } else {" << endl;
fcc9bb6f 1300 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1301 out << " gSystem->ChangeDirectory(cdir);" << endl;
1302 out << " return kFALSE;" << endl;
1303 out << " }" << endl;
1304 out << " // Check for SETUP.C and execute" << endl;
1305 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1306 out << " printf(\"*******************************\\n\");" << endl;
1307 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1308 out << " printf(\"*******************************\\n\");" << endl;
1309 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1310 out << " } else {" << endl;
fcc9bb6f 1311 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1312 out << " gSystem->ChangeDirectory(cdir);" << endl;
1313 out << " return kFALSE;" << endl;
1314 out << " }" << endl;
1315 out << " // Restore original workdir" << endl;
1316 out << " gSystem->ChangeDirectory(cdir);" << endl;
1317 out << " return kTRUE;" << endl;
1318 out << "}" << endl;
1319 }
c57f56b7 1320 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1321 }
1322 Bool_t copy = kTRUE;
1323 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1324 if (copy) {
1325 CdWork();
1326 TString workdir = gGrid->GetHomeDirectory();
1327 workdir += fGridWorkingDir;
1328 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1329 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1330 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1331 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1332 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1333 }
1334 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1335 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1336 }
1337}
1338
1339//______________________________________________________________________________
1340void AliAnalysisAlien::WriteExecutable()
1341{
1342// Generate the alien executable script.
1343 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1344 ofstream out;
1345 out.open(fExecutable.Data(), ios::out);
1346 if (out.bad()) {
1347 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1348 return;
1349 }
1350 out << "#!/bin/bash" << endl;
1351 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1352 out << "echo \"=========================================\"" << endl;
1353 out << "echo \"############## PATH : ##############\"" << endl;
1354 out << "echo $PATH" << endl;
1355 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1356 out << "echo $LD_LIBRARY_PATH" << endl;
1357 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1358 out << "echo $ROOTSYS" << endl;
1359 out << "echo \"############## which root : ##############\"" << endl;
1360 out << "which root" << endl;
1361 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1362 out << "echo $ALICE_ROOT" << endl;
1363 out << "echo \"############## which aliroot : ##############\"" << endl;
1364 out << "which aliroot" << endl;
1365 out << "echo \"=========================================\"" << endl << endl;
1366// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1367 out << "root -b -q ";
1368 out << fAnalysisMacro.Data() << endl << endl;
1369 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1370 }
1371 Bool_t copy = kTRUE;
1372 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1373 if (copy) {
1374 CdWork();
1375 TString workdir = gGrid->GetHomeDirectory();
1376 workdir += fGridWorkingDir;
1377 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1378 if (FileExists(executable)) gGrid->Rm(executable);
1379 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1380 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1381 }
1382}
1383
1384//______________________________________________________________________________
1385void AliAnalysisAlien::WriteValidationScript()
1386{
1387// Generate the alien validation script.
1388 // Generate the validation script
1389 TObjString *os;
1390 if (!Connect()) {
1391 Error("WriteValidationScript", "Alien connection required");
1392 return;
1393 }
1394 TString out_stream = "";
1395 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1396 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1397 ofstream out;
1398 out.open("validate.sh", ios::out);
1399 out << "#!/bin/bash" << endl;
1400 out << "##################################################" << endl;
1401 out << "validateout=`dirname $0`" << endl;
1402 out << "validatetime=`date`" << endl;
1403 out << "validated=\"0\";" << endl;
1404 out << "error=0" << endl;
1405 out << "if [ -z $validateout ]" << endl;
1406 out << "then" << endl;
1407 out << " validateout=\".\"" << endl;
1408 out << "fi" << endl << endl;
1409 out << "cd $validateout;" << endl;
1410 out << "validateworkdir=`pwd`;" << endl << endl;
1411 out << "echo \"*******************************************************\"" << out_stream << endl;
1412 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1413 out << "" << endl;
1414 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1415 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1416 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1417 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1418 out << "ls -la ./" << out_stream << endl;
1419 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1420 out << "##################################################" << endl;
1421 TObjArray *arr = fOutputFiles.Tokenize(" ");
1422 TIter next1(arr);
1423 TString output_file;
1424 while ((os=(TObjString*)next1())) {
1425 output_file = os->GetString();
1426 Int_t index = output_file.Index("@");
1427 if (index > 0) output_file.Remove(index);
1428 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1429 out << " error=1" << endl;
1430 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1431 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1432 out << "fi" << endl;
1433 }
1434 delete arr;
1435 out << "if [ $error = 0 ] ; then" << endl;
1436 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1437 out << "fi" << endl;
1438
1439 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1440 out << "echo \"*******************************************************\"" << out_stream << endl;
1441 out << "cd -" << endl;
1442 out << "exit $error" << endl;
1443 }
1444 Bool_t copy = kTRUE;
1445 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1446 if (copy) {
1447 CdWork();
1448 TString workdir = gGrid->GetHomeDirectory();
1449 workdir += fGridWorkingDir;
1450 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1451 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1452 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1453 }
1454}