]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bug fixed
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
35#include "AliAnalysisAlien.h"
36
37ClassImp(AliAnalysisAlien)
38
39//______________________________________________________________________________
40AliAnalysisAlien::AliAnalysisAlien()
41 :AliAnalysisGrid(),
42 fGridJDL(NULL),
43 fPrice(0),
44 fTTL(0),
45 fSplitMaxInputFileNumber(0),
46 fMaxInitFailed(0),
47 fMasterResubmitThreshold(0),
48 fRunNumbers(),
49 fExecutable(),
50 fArguments(),
51 fAnalysisMacro(),
52 fAnalysisSource(),
53 fAdditionalLibs(),
54 fSplitMode(),
55 fAPIVersion(),
56 fROOTVersion(),
57 fAliROOTVersion(),
58 fUser(),
59 fGridWorkingDir(),
60 fGridDataDir(),
61 fDataPattern(),
62 fGridOutputDir(),
63 fOutputArchive(),
64 fOutputFiles(),
65 fInputFormat(),
e7c71df0 66 fDatasetName(),
c57f56b7 67 fJDLName(),
4e5c5506 68 fInputFiles(0),
69 fPackages(0)
c57f56b7 70{
71// Dummy ctor.
72 SetDefaults();
73}
74
75//______________________________________________________________________________
76AliAnalysisAlien::AliAnalysisAlien(const char *name)
77 :AliAnalysisGrid(name),
78 fGridJDL(NULL),
79 fPrice(0),
80 fTTL(0),
81 fSplitMaxInputFileNumber(0),
82 fMaxInitFailed(0),
83 fMasterResubmitThreshold(0),
84 fRunNumbers(),
85 fExecutable(),
86 fArguments(),
87 fAnalysisMacro(),
88 fAnalysisSource(),
89 fAdditionalLibs(),
90 fSplitMode(),
91 fAPIVersion(),
92 fROOTVersion(),
93 fAliROOTVersion(),
94 fUser(),
95 fGridWorkingDir(),
96 fGridDataDir(),
97 fDataPattern(),
98 fGridOutputDir(),
99 fOutputArchive(),
100 fOutputFiles(),
101 fInputFormat(),
e7c71df0 102 fDatasetName(),
c57f56b7 103 fJDLName(),
4e5c5506 104 fInputFiles(0),
105 fPackages(0)
c57f56b7 106{
107// Default ctor.
108 SetDefaults();
109}
110
111//______________________________________________________________________________
112AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
113 :AliAnalysisGrid(other),
114 fGridJDL(NULL),
115 fPrice(other.fPrice),
116 fTTL(other.fTTL),
117 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
118 fMaxInitFailed(other.fMaxInitFailed),
119 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
120 fRunNumbers(other.fRunNumbers),
121 fExecutable(other.fExecutable),
122 fArguments(other.fArguments),
123 fAnalysisMacro(other.fAnalysisMacro),
124 fAnalysisSource(other.fAnalysisSource),
125 fAdditionalLibs(other.fAdditionalLibs),
126 fSplitMode(other.fSplitMode),
127 fAPIVersion(other.fAPIVersion),
128 fROOTVersion(other.fROOTVersion),
129 fAliROOTVersion(other.fAliROOTVersion),
130 fUser(other.fUser),
131 fGridWorkingDir(other.fGridWorkingDir),
132 fGridDataDir(other.fGridDataDir),
133 fDataPattern(other.fDataPattern),
134 fGridOutputDir(other.fGridOutputDir),
135 fOutputArchive(other.fOutputArchive),
136 fOutputFiles(other.fOutputFiles),
137 fInputFormat(other.fInputFormat),
e7c71df0 138 fDatasetName(other.fDatasetName),
c57f56b7 139 fJDLName(other.fJDLName),
4e5c5506 140 fInputFiles(0),
141 fPackages(0)
c57f56b7 142{
143// Copy ctor.
144 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
145 if (other.fInputFiles) {
146 fInputFiles = new TObjArray();
147 TIter next(other.fInputFiles);
148 TObject *obj;
149 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
150 fInputFiles->SetOwner();
151 }
4e5c5506 152 if (other.fPackages) {
153 fPackages = new TObjArray();
154 TIter next(other.fPackages);
155 TObject *obj;
156 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
157 fPackages->SetOwner();
158 }
c57f56b7 159}
160
161//______________________________________________________________________________
162AliAnalysisAlien::~AliAnalysisAlien()
163{
164// Destructor.
165 if (fGridJDL) delete fGridJDL;
166 if (fInputFiles) delete fInputFiles;
4e5c5506 167 if (fPackages) delete fPackages;
c57f56b7 168}
169
170//______________________________________________________________________________
171AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
172{
173// Assignment.
174 if (this != &other) {
175 AliAnalysisGrid::operator=(other);
176 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
177 fPrice = other.fPrice;
178 fTTL = other.fTTL;
179 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
180 fMaxInitFailed = other.fMaxInitFailed;
181 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
182 fRunNumbers = other.fRunNumbers;
183 fExecutable = other.fExecutable;
184 fArguments = other.fArguments;
185 fAnalysisMacro = other.fAnalysisMacro;
186 fAnalysisSource = other.fAnalysisSource;
187 fAdditionalLibs = other.fAdditionalLibs;
188 fSplitMode = other.fSplitMode;
189 fAPIVersion = other.fAPIVersion;
190 fROOTVersion = other.fROOTVersion;
191 fAliROOTVersion = other.fAliROOTVersion;
192 fUser = other.fUser;
193 fGridWorkingDir = other.fGridWorkingDir;
194 fGridDataDir = other.fGridDataDir;
195 fDataPattern = other.fDataPattern;
196 fGridOutputDir = other.fGridOutputDir;
197 fOutputArchive = other.fOutputArchive;
198 fOutputFiles = other.fOutputFiles;
199 fInputFormat = other.fInputFormat;
e7c71df0 200 fDatasetName = other.fDatasetName;
c57f56b7 201 fJDLName = other.fJDLName;
202 if (other.fInputFiles) {
203 fInputFiles = new TObjArray();
204 TIter next(other.fInputFiles);
205 TObject *obj;
206 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
207 fInputFiles->SetOwner();
208 }
4e5c5506 209 if (other.fPackages) {
210 fPackages = new TObjArray();
211 TIter next(other.fPackages);
212 TObject *obj;
213 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
214 fPackages->SetOwner();
215 }
c57f56b7 216 }
217 return *this;
218}
219
220//______________________________________________________________________________
221void AliAnalysisAlien::AddRunNumber(Int_t run)
222{
223// Add a run number to the list of runs to be processed.
224 if (fRunNumbers.Length()) fRunNumbers += " ";
225 fRunNumbers += Form("%d", run);
226}
227
228//______________________________________________________________________________
229void AliAnalysisAlien::AddDataFile(const char *lfn)
230{
231// Adds a data file to the input to be analysed. The file should be a valid LFN
232// or point to an existing file in the alien workdir.
233 if (!fInputFiles) fInputFiles = new TObjArray();
234 fInputFiles->Add(new TObjString(lfn));
235}
236
237//______________________________________________________________________________
238Bool_t AliAnalysisAlien::Connect()
239{
240// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
241 if (gGrid && gGrid->IsConnected()) return kTRUE;
242 if (!gSystem->Getenv("alien_API_USER")) {
243 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
244 gSystem->Getenv("UID"));
245 return kFALSE;
246 }
247 if (!gGrid) {
248 Info("Connect", "Trying to connect to AliEn ...");
249 TGrid::Connect("alien://");
250 }
251 if (!gGrid || !gGrid->IsConnected()) {
252 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
253 return kFALSE;
254 }
255 fUser = gGrid->GetUser();
256 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
257 return kTRUE;
258}
259
260//______________________________________________________________________________
261void AliAnalysisAlien::CdWork()
262{
263// Check validity of alien workspace. Create directory if possible.
264 if (!Connect()) {
265 Error("CdWork", "Alien connection required");
266 return;
267 }
268 TString homedir = gGrid->GetHomeDirectory();
269 TString workdir = homedir + fGridWorkingDir;
270 if (!gGrid->Cd(workdir)) {
271 gGrid->Cd(homedir);
272 if (gGrid->Mkdir(workdir)) {
273 gGrid->Cd(fGridWorkingDir);
274 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
275 } else {
276 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
277 workdir.Data(), homedir.Data());
278 fGridWorkingDir = "";
279 }
280 }
281}
282
283//______________________________________________________________________________
284Bool_t AliAnalysisAlien::CheckInputData()
285{
286// Check validity of input data. If necessary, create xml files.
287 if (!fInputFiles && !fRunNumbers.Length()) {
288 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
289 return kFALSE;
290 }
291 // Process declared files
292 Bool_t is_collection = kFALSE;
293 Bool_t is_xml = kFALSE;
294 Bool_t use_tags = kFALSE;
295 Bool_t checked = kFALSE;
296 CdWork();
297 TString file;
298 TString workdir = gGrid->GetHomeDirectory();
299 workdir += fGridWorkingDir;
300 if (fInputFiles) {
301 TObjString *objstr;
302 TIter next(fInputFiles);
303 while ((objstr=(TObjString*)next())) {
304 file = workdir;
305 file += "/";
306 file += objstr->GetString();
307 // Store full lfn path
308 if (FileExists(file)) objstr->SetString(file);
309 else {
310 file = objstr->GetName();
311 if (!FileExists(objstr->GetName())) {
312 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
313 objstr->GetName(), workdir.Data());
314 return kFALSE;
315 }
316 }
317 Bool_t iscoll, isxml, usetags;
318 CheckDataType(file, iscoll, isxml, usetags);
319 if (!checked) {
320 checked = kTRUE;
321 is_collection = iscoll;
322 is_xml = isxml;
323 use_tags = usetags;
324 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
325 } else {
326 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
327 Error("CheckInputData", "Some conflict was found in the types of inputs");
328 return kFALSE;
329 }
330 }
331 }
332 }
333 // Process requested run numbers
334 if (!fRunNumbers.Length()) return kTRUE;
335 // Check validity of alien data directory
336 if (!fGridDataDir.Length()) {
337 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
338 return kFALSE;
339 }
340 if (!gGrid->Cd(fGridDataDir)) {
341 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
342 return kFALSE;
343 }
344 if (is_collection) {
345 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
346 return kFALSE;
347 }
348
349 if (checked && !is_xml) {
350 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
351 return kFALSE;
352 }
353 // Check validity of run number(s)
354 TObjArray *arr;
355 TObjString *os;
356 TString path;
357 if (!checked) {
358 checked = kTRUE;
359 use_tags = fDataPattern.Contains("tag");
360 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
361 }
362 if (use_tags != fDataPattern.Contains("tag")) {
363 Error("CheckInputData", "Cannot mix input files using/not using tags");
364 return kFALSE;
365 }
366 if (fRunNumbers.Length()) {
367 arr = fRunNumbers.Tokenize(" ");
368 TIter next(arr);
369 while ((os=(TObjString*)next())) {
370 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
371 if (!gGrid->Cd(path)) {
372 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
373 return kFALSE;
374 }
375 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
376 TString msg = "\n##### file: ";
377 msg += path;
378 msg += " type: xml_collection;";
379 if (use_tags) msg += " using_tags: Yes";
380 else msg += " using_tags: No";
381 Info("CheckDataType", msg.Data());
382 AddDataFile(path);
383 }
384 delete arr;
385 }
386 return kTRUE;
387}
388
389//______________________________________________________________________________
390Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
391{
392// Create dataset for the grid data directory + run number.
393 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
394 if (!Connect()) {
395 Error("CreateDataset", "Cannot create dataset with no grid connection");
396 return kFALSE;
397 }
398
399 // Cd workspace
400 CdWork();
401 TString workdir = gGrid->GetHomeDirectory();
402 workdir += fGridWorkingDir;
403
404 // Compose the 'find' command arguments
405 TString command;
406 TString options = "-x collection ";
6da75e0b 407 if (TestBit(AliAnalysisGrid::kTest)) options += "-l 100 ";
c57f56b7 408 TString conditions = "";
409
410 TString file;
411 TString path;
412 if (!fRunNumbers.Length()) return kTRUE;
413 // Several runs
414 TObjArray *arr = fRunNumbers.Tokenize(" ");
415 TObjString *os;
416 TIter next(arr);
417 while ((os=(TObjString*)next())) {
418 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
1535db97 419 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
420 else file = Form("%s.xml", os->GetString().Data());
421 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
c57f56b7 422 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
423 gGrid->Rm(file);
424 }
425 command = "find ";
426 command += options;
427 command += path;
428 command += pattern;
1535db97 429// conditions = Form(" > %s", file.Data());
c57f56b7 430 command += conditions;
431 TGridResult *res = gGrid->Command(command);
432 if (res) delete res;
1535db97 433 // Write standard output to file
434 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
435 if (TestBit(AliAnalysisGrid::kTest)) break;
436 // Copy xml file to alien space
437 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
c57f56b7 438 if (!FileExists(file)) {
439 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
440 delete arr;
441 return kFALSE;
442 }
c57f56b7 443 }
444 delete arr;
c57f56b7 445 return kTRUE;
446}
447
448//______________________________________________________________________________
449Bool_t AliAnalysisAlien::CreateJDL()
450{
451// Generate a JDL file according to current settings. The name of the file is
452// specified by fJDLName.
453 Bool_t error = kFALSE;
454 TObjArray *arr = 0;
455 Bool_t copy = kTRUE;
456 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
457 Bool_t generate = kTRUE;
458 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
459 if (!Connect()) {
460 Error("CreateJDL", "Alien connection required");
461 return kFALSE;
462 }
463 // Check validity of alien workspace
464 CdWork();
465 TString workdir = gGrid->GetHomeDirectory();
466 workdir += fGridWorkingDir;
467 if (generate) {
468 TObjString *os;
469 if (!fInputFiles) {
470 Error("CreateJDL()", "Define some input files for your analysis.");
471 error = kTRUE;
472 }
473 // Compose list of input files
474 // Check if output files were defined
475 if (!fOutputFiles.Length()) {
476 Error("CreateJDL", "You must define at least one output file");
477 error = kTRUE;
478 }
479 // Check if an output directory was defined and valid
480 if (!fGridOutputDir.Length()) {
481 Error("CreateJDL", "You must define AliEn output directory");
482 error = kTRUE;
483 } else {
484 if (!gGrid->Cd(fGridOutputDir)) {
485 if (gGrid->Mkdir(fGridOutputDir)) {
486 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
487 } else {
488 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
489 error = kTRUE;
490 }
491 }
492 gGrid->Cd(workdir);
493 }
494 // Exit if any error up to now
495 if (error) return kFALSE;
496 // Set JDL fields
497 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
498 fGridJDL->SetExecutable(fExecutable);
499// fGridJDL->SetTTL((UInt_t)fTTL);
500 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
501 if (fMaxInitFailed > 0)
502 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
503 if (fSplitMaxInputFileNumber > 0)
504 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
505 if (fSplitMode.Length())
506 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
507// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
508 if (fAliROOTVersion.Length())
509 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
510 if (fROOTVersion.Length())
511 fGridJDL->AddToPackages("ROOT", fROOTVersion);
512 if (fAPIVersion.Length())
513 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
514 fGridJDL->SetInputDataListFormat(fInputFormat);
515 fGridJDL->SetInputDataList("wn.xml");
516 if (fInputFiles) {
517 TIter next(fInputFiles);
518 while ((os=(TObjString*)next()))
519 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
520 }
521 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
522 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
523 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
524 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
525 if (fAdditionalLibs.Length()) {
526 arr = fAdditionalLibs.Tokenize(" ");
527 TIter next(arr);
528 while ((os=(TObjString*)next())) {
529 if (os->GetString().Contains(".so")) continue;
530 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
531 }
532 delete arr;
533 }
4e5c5506 534 if (fPackages) {
535 TIter next(fPackages);
536 TObject *obj;
537 while ((obj=next()))
538 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
539 }
c57f56b7 540 if (fOutputArchive.Length()) {
541 arr = fOutputArchive.Tokenize(" ");
542 TIter next(arr);
543 while ((os=(TObjString*)next()))
544 fGridJDL->AddToOutputArchive(os->GetString().Data());
545 delete arr;
546 }
547 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
548 arr = fOutputFiles.Tokenize(" ");
549 TIter next(arr);
550 while ((os=(TObjString*)next())) fGridJDL->AddToOutputSandbox(os->GetString());
551 delete arr;
552// fGridJDL->SetPrice((UInt_t)fPrice);
553 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
554 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
555 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
556 // Generate the JDL as a string
557 TString sjdl = fGridJDL->Generate();
558 Int_t index;
559 index = sjdl.Index("Executable");
560 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
561 index = sjdl.Index("Split ");
562 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
563 index = sjdl.Index("SplitMaxInputFileNumber");
564 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
565 index = sjdl.Index("InputDataCollection");
566 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
567 index = sjdl.Index("InputFile");
568 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
569 index = sjdl.Index("InputDataList ");
570 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
571 index = sjdl.Index("InputDataListFormat");
572 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
573 index = sjdl.Index("Price");
574 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
575 index = sjdl.Index("Requirements");
576 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
577 index = sjdl.Index("Packages");
578 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
579 index = sjdl.Index("User");
580 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
581 index = sjdl.Index("TTL");
582 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
583 index = sjdl.Index("OutputFile");
584 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
585 index = sjdl.Index("OutputDir");
586 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
587 index = sjdl.Index("OutputArchive");
588 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
589 index = sjdl.Index("MaxInitFailed");
590 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
591 index = sjdl.Index("MasterResubmitThreshold");
592 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
593 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
594 index = sjdl.Index("Validationcommand");
595 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
596 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
597 sjdl.ReplaceAll("(member", "\n (member");
598 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
599 sjdl.ReplaceAll("{", "{\n ");
600 sjdl.ReplaceAll("};", "\n};");
601 sjdl.ReplaceAll("{\n \n", "{\n");
602 sjdl.ReplaceAll("\n\n", "\n");
603 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
604 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
605 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
606 index = sjdl.Index("JDLVariables");
607 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
608 // Write jdl to file
609 ofstream out;
610 out.open(fJDLName.Data(), ios::out);
611 if (out.bad()) {
612 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
613 return kFALSE;
614 }
615 out << sjdl << endl;
616 }
617 // Copy jdl to grid workspace
618 if (!copy) {
619 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
620 } else {
621 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
622 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
623 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
624 if (fAdditionalLibs.Length()) {
e7c71df0 625 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 626 TObjString *os;
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
c57f56b7 629 if (os->GetString().Contains(".so")) continue;
4e5c5506 630 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 631 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
632 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
633 }
634 delete arr;
635 }
4e5c5506 636 if (fPackages) {
637 TIter next(fPackages);
638 TObject *obj;
639 while ((obj=next())) {
640 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
641 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
642 }
643 }
c57f56b7 644 }
645 return kTRUE;
646}
647
648//______________________________________________________________________________
649Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
650{
651// Returns true if file exists.
652 if (!gGrid) {
653 Error("FileExists", "No connection to grid");
654 return kFALSE;
655 }
656 TGridResult *res = gGrid->Ls(lfn);
657 if (!res) return kFALSE;
658 TMap *map = dynamic_cast<TMap*>(res->At(0));
659 if (!map) {
660 delete res;
661 return kFALSE;
662 }
663 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
664 if (!objs || !objs->GetString().Length()) {
665 delete res;
666 return kFALSE;
667 }
668 delete res;
669 return kTRUE;
670}
671
672//______________________________________________________________________________
673void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
674{
675// Check input data type.
676 is_collection = kFALSE;
677 is_xml = kFALSE;
678 use_tags = kFALSE;
679 if (!gGrid) {
680 Error("CheckDataType", "No connection to grid");
681 return;
682 }
683 is_collection = IsCollection(lfn);
684 TString msg = "\n##### file: ";
685 msg += lfn;
686 if (is_collection) {
687 msg += " type: raw_collection;";
688 // special treatment for collections
689 is_xml = kFALSE;
690 // check for tag files in the collection
691 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
692 if (!res) {
693 msg += " using_tags: No (unknown)";
694 Info("CheckDataType", msg.Data());
695 return;
696 }
697 const char* typeStr = res->GetKey(0, "origLFN");
698 if (!typeStr || !strlen(typeStr)) {
699 msg += " using_tags: No (unknown)";
700 Info("CheckDataType", msg.Data());
701 return;
702 }
703 TString file = typeStr;
704 use_tags = file.Contains(".tag");
705 if (use_tags) msg += " using_tags: Yes";
706 else msg += " using_tags: No";
707 Info("CheckDataType", msg.Data());
708 return;
709 }
710 TString slfn(lfn);
711 slfn.ToLower();
712 is_xml = slfn.Contains(".xml");
713 if (is_xml) {
714 // Open xml collection and check if there are tag files inside
715 msg += " type: xml_collection;";
716 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
717 if (!coll) {
718 msg += " using_tags: No (unknown)";
719 Info("CheckDataType", msg.Data());
720 return;
721 }
722 TMap *map = coll->Next();
723 if (!map) {
724 msg += " using_tags: No (unknown)";
725 Info("CheckDataType", msg.Data());
726 return;
727 }
728 map = (TMap*)map->GetValue("");
729 TString file;
730 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
731 use_tags = file.Contains(".tag");
732 delete coll;
733 if (use_tags) msg += " using_tags: Yes";
734 else msg += " using_tags: No";
735 Info("CheckDataType", msg.Data());
736 return;
737 }
738 use_tags = slfn.Contains(".tag");
739 if (slfn.Contains(".root")) msg += " type: root file;";
740 else msg += " type: unhnown file;";
741 if (use_tags) msg += " using_tags: Yes";
742 else msg += " using_tags: No";
743 Info("CheckDataType", msg.Data());
744}
745
4e5c5506 746//______________________________________________________________________________
747void AliAnalysisAlien::EnablePackage(const char *package)
748{
749// Enables a par file supposed to exist in the current directory.
750 TString pkg(package);
751 pkg.ReplaceAll(".par", "");
752 pkg += ".par";
753 if (gSystem->AccessPathName(pkg)) {
754 Error("EnablePackage", "Package %s not found", pkg.Data());
755 return;
756 }
757 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
758 Info("EnablePackage", "AliEn plugin will use .par packages");
759 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
760 if (!fPackages) {
761 fPackages = new TObjArray();
762 fPackages->SetOwner();
763 }
764 fPackages->Add(new TObjString(pkg));
765}
766
c57f56b7 767//______________________________________________________________________________
768Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
769{
770// Returns true if file is a collection. Functionality duplicated from
771// TAlien::Type() because we don't want to directly depend on TAlien.
772 if (!gGrid) {
773 Error("IsCollection", "No connection to grid");
774 return kFALSE;
775 }
776 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
777 if (!res) return kFALSE;
778 const char* typeStr = res->GetKey(0, "type");
779 if (!typeStr || !strlen(typeStr)) return kFALSE;
780 if (!strcmp(typeStr, "collection")) return kTRUE;
781 delete res;
782 return kFALSE;
783}
784
785//______________________________________________________________________________
786void AliAnalysisAlien::SetDefaults()
787{
788// Set default values for everything. What cannot be filled will be left empty.
789 if (fGridJDL) delete fGridJDL;
790 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
791 fPrice = 1;
792 fTTL = 30000;
793 fSplitMaxInputFileNumber = 100;
794 fMaxInitFailed = 0;
795 fMasterResubmitThreshold = 0;
796 fRunNumbers = "";
797 fExecutable = "analysis.sh";
798 fArguments = "";
799 fAnalysisMacro = "myAnalysis.C";
800 fAnalysisSource = "";
801 fAdditionalLibs = "";
802 fSplitMode = "se";
803 fAPIVersion = "";
804 fROOTVersion = "";
805 fAliROOTVersion = "";
806 fUser = ""; // Your alien user name
807 fGridWorkingDir = "";
808 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
809 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
810 fGridOutputDir = "output";
811 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
812 fOutputFiles = ""; // Like "AliAODs.root histos.root"
813 fInputFormat = "xml-single";
814 fJDLName = "analysis.jdl";
815}
816
817//______________________________________________________________________________
818Bool_t AliAnalysisAlien::MergeOutputs()
819{
820// Merge analysis outputs existing in the AliEn space.
821 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
822 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
823 if (!Connect()) {
824 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
825 return kFALSE;
826 }
827 // Get the output path
828 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
829 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
830 if (!gGrid->Cd(output)) {
831 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
832 return kFALSE;
833 }
834 if (!fOutputFiles.Length()) {
835 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
836 return kFALSE;
837 }
838 TObjArray *list = fOutputFiles.Tokenize(" ");
839 TIter next(list);
840 TObjString *str;
841 TString command;
842 TString output_file;
843 Bool_t merged = kTRUE;
844 while((str=(TObjString*)next())) {
845 output_file = str->GetString();
846 Int_t index = output_file.Index("@");
847 if (index > 0) output_file.Remove(index);
848 command = Form("find %s/ *%s", output.Data(), output_file.Data());
849 printf("command: %s\n", command.Data());
850 TGridResult *res = gGrid->Command(command);
851 if (!res) continue;
852 TFileMerger *fm = 0;
853 TIter nextmap(res);
854 TMap *map;
855 while ((map=(TMap*)nextmap())) {
856 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
857 if (!objs || !objs->GetString().Length()) {
858 delete res;
859 continue;
860 }
861 if (!fm) {
862 fm = new TFileMerger(kFALSE);
863 fm->SetFastMethod(kTRUE);
864 fm->OutputFile(output_file);
865 }
866 fm->AddFile(objs->GetString());
867 }
868 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
869 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
870 merged = kFALSE;
871 delete res;
872 continue;
873 }
874 if (!fm->Merge()) {
875 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
876 merged = kFALSE;
877 } else {
878 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
879 }
880 delete fm;
881 delete res;
882 }
883 if (!merged) {
884 Error("MergeOutputs", "Terminate() will NOT be executed");
885 }
886 return merged;
887}
888
889//______________________________________________________________________________
890void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
891{
892// Start remote grid analysis.
893
894 if (TestBit(AliAnalysisGrid::kOffline)) {
895 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
896 \n there nor any job run. You can revise the JDL and analysis \
897 \n macro then run the same in \"submit\" mode.");
898 } else if (TestBit(AliAnalysisGrid::kTest)) {
899 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
900 \n dataset.");
901 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
902 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
903 \n space and job submitted.");
904 } else if (TestBit(AliAnalysisGrid::kMerge)) {
905 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
906 return;
907 } else {
908 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
909 }
910
911 if (!Connect()) {
912 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
913 return;
914 }
915 if (!CheckInputData()) {
916 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
917 return;
918 }
919 CreateDataset(fDataPattern);
920 WriteAnalysisFile();
921 WriteAnalysisMacro();
922 WriteExecutable();
923 WriteValidationScript();
924 if (!CreateJDL()) return;
925 if (TestBit(AliAnalysisGrid::kOffline)) return;
926 if (TestBit(AliAnalysisGrid::kTest)) {
927 // Locally testing the analysis
928 Info("StartAnalysis", "\n_______________________________________________________________________ \
929 \n Running analysis script in a daughter shell as on a worker node \
930 \n_______________________________________________________________________");
931 TObjArray *list = fOutputFiles.Tokenize(" ");
932 TIter next(list);
933 TObjString *str;
934 TString output_file;
935 while((str=(TObjString*)next())) {
936 output_file = str->GetString();
937 Int_t index = output_file.Index("@");
938 if (index > 0) output_file.Remove(index);
939 gSystem->Exec(Form("rm %s", output_file.Data()));
940 }
941 delete list;
942 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
943 gSystem->Exec("bash validate.sh");
944// gSystem->Exec("cat stdout");
945 return;
946 }
947 // Submit AliEn job
948 CdWork();
949 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
950 TString jobID = "";
951 if (res) {
952 const char *cjobId = res->GetKey(0,"jobId");
953 if (!cjobId) {
954 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
955 return;
956 } else {
957 Info("StartAnalysis", "\n_______________________________________________________________________ \
958 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
959 \n_______________________________________________________________________",
960 fJDLName.Data(), cjobId);
961 jobID = cjobId;
962 }
963 delete res;
964 }
965 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
966 \n You may exit at any time and terminate the job later using the option <terminate> \
967 \n ##################################################################################", jobID.Data());
968 gGrid->Shell();
969}
970
971//______________________________________________________________________________
972void AliAnalysisAlien::WriteAnalysisFile()
973{
974// Write current analysis manager into the file analysis.root
975 if (!TestBit(AliAnalysisGrid::kSubmit)) {
976 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
977 if (!mgr || !mgr->IsInitialized()) {
978 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
979 return;
980 }
981 // Check analysis type
982 TObject *handler;
983 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
984 handler = (TObject*)mgr->GetInputEventHandler();
985 if (handler) {
986 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
987 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
988 }
989 TDirectory *cdir = gDirectory;
990 TFile *file = TFile::Open("analysis.root", "RECREATE");
991 if (file) {
992 mgr->Write();
993 delete file;
994 }
995 if (cdir) cdir->cd();
996 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
997 }
998 Bool_t copy = kTRUE;
999 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1000 if (copy) {
1001 CdWork();
1002 TString workdir = gGrid->GetHomeDirectory();
1003 workdir += fGridWorkingDir;
1004 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1005 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1006 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1007 }
1008}
1009
1010//______________________________________________________________________________
1011void AliAnalysisAlien::WriteAnalysisMacro()
1012{
1013// Write the analysis macro that will steer the analysis in grid mode.
1014 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1015 ofstream out;
1016 out.open(fAnalysisMacro.Data(), ios::out);
1017 if (!out.good()) {
1018 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1019 return;
1020 }
1021 TString func = fAnalysisMacro;
1022 TString type = "ESD";
1023 TString comment = "// Analysis using ";
1024 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1025 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1026 type = "AOD";
1027 comment += "AOD";
1028 }
1029 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1030 else comment += " data";
1031 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1032 func.ReplaceAll(".C", "");
1033 out << "void " << func.Data() << "()" << endl;
1034 out << "{" << endl;
1035 out << comment.Data() << endl;
1036 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1037 out << "// load base root libraries" << endl;
1038 out << " gSystem->Load(\"libTree\");" << endl;
1039 out << " gSystem->Load(\"libGeom\");" << endl;
1040 out << " gSystem->Load(\"libVMC\");" << endl;
1041 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4e5c5506 1042 if (!fPackages) {
1043 out << "// Load analysis framework libraries" << endl;
1044 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1045 out << " gSystem->Load(\"libESD\");" << endl;
1046 out << " gSystem->Load(\"libAOD\");" << endl;
1047 out << " gSystem->Load(\"libANALYSIS\");" << endl;
1048 out << " gSystem->Load(\"libANALYSISalice\");" << endl << endl;
1049 out << "// include path (remove if using par files)" << endl;
1050 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl << endl;
1051 } else {
1052 out << "// Compile all par packages" << endl;
1053 TIter next(fPackages);
1054 TObject *obj;
1055 while ((obj=next()))
1056 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1057 }
6da75e0b 1058 if (fAdditionalLibs.Length()) {
1059 out << "// Add aditional AliRoot libraries" << endl;
1060 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1061 TIter next(list);
1062 TObjString *str;
1063 while((str=(TObjString*)next())) {
1064 if (str->GetString().Contains(".so"))
1065 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1066 }
1067 if (list) delete list;
1068 }
1069 out << endl;
c57f56b7 1070 out << "// analysis source to be compiled at runtime (if any)" << endl;
1071 if (fAnalysisSource.Length()) {
1072 TObjArray *list = fAnalysisSource.Tokenize(" ");
1073 TIter next(list);
1074 TObjString *str;
1075 while((str=(TObjString*)next())) {
1076 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1077 }
1078 if (list) delete list;
1079 }
1080 out << endl;
1081 out << "// connect to AliEn and make the chain" << endl;
1082 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1083 if (IsUsingTags()) {
1084 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1085 } else {
1086 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1087 }
1088 out << "// read the analysis manager from file" << endl;
1089 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1090 out << " if (!file) return;" << endl;
1091 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1092 out << " AliAnalysisManager *mgr = 0;" << endl;
1093 out << " TKey *key;" << endl;
1094 out << " while ((key=(TKey*)nextkey())) {" << endl;
1095 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1096 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1097 out << " };" << endl;
1098 out << " if (!mgr) {" << endl;
1099 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1100 out << " return;" << endl;
1101 out << " }" << endl << endl;
1102 out << " mgr->PrintStatus();" << endl;
1103 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1104 out << "}" << endl << endl;
1105 if (IsUsingTags()) {
1106 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1107 out << "{" << endl;
1108 out << "// Create a chain using tags from the xml file." << endl;
1109 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1110 out << " if (!coll) {" << endl;
1111 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1112 out << " return NULL;" << endl;
1113 out << " }" << endl;
1114 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1115 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1116 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1117 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1118 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1119 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1120 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1121 out << " // Check if the cuts configuration file was provided" << endl;
1122 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1123 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1124 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1125 out << " }" << endl;
1126 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1127 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1128 out << " chain->ls();" << endl;
1129 out << " return chain;" << endl;
1130 out << "}" << endl;
1131 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1132 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1133 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1134 msg += " AliLHCTagCuts *lhcCuts,\n";
1135 msg += " AliDetectorTagCuts *detCuts,\n";
1136 msg += " AliEventTagCuts *evCuts)";
1137 Info("WriteAnalysisMacro", msg.Data());
1138 }
1139 } else {
1140 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1141 out << "{" << endl;
1142 out << "// Create a chain using url's from xml file" << endl;
1143 out << " TString treename = type;" << endl;
1144 out << " treename.ToLower();" << endl;
1145 out << " treename += \"Tree\";" << endl;
1146 out << " printf(\"***************************************\");" << endl;
1147 out << " printf(\" Getting chain of trees %s\\n\", treename);" << endl;
1148 out << " printf(\"***************************************\");" << endl;
1149 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1150 out << " if (!coll) {" << endl;
1151 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1152 out << " return NULL;" << endl;
1153 out << " }" << endl;
1154 out << " TChain *chain = new TChain(treename);" << endl;
1155 out << " coll->Reset();" << endl;
1156 out << " while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
1157 out << " if (!chain->GetNtrees()) {" << endl;
1158 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1159 out << " return NULL;" << endl;
1160 out << " }" << endl;
1161 out << " return chain;" << endl;
1162 out << "}" << endl;
1163 }
4e5c5506 1164 if (fPackages) {
1165 out << "Bool_t SetupPar(const char *package) {" << endl;
1166 out << "// Compile the package and set it up." << endl;
1167 out << " TString pkgdir = package;" << endl;
1168 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
1169 out << " gSystem->Exec(Form(\"tar xvzf %s\", package));" << endl;
1170 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1171 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1172 out << " // Check for BUILD.sh and execute" << endl;
1173 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1174 out << " printf(\"*******************************\\n\");" << endl;
1175 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1176 out << " printf(\"*******************************\\n\");" << endl;
1177 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
1178 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", package);" << endl;
1179 out << " gSystem->ChangeDirectory(cdir);" << endl;
1180 out << " return kFALSE;" << endl;
1181 out << " }" << endl;
1182 out << " } else {" << endl;
1183 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", package);" << endl;
1184 out << " gSystem->ChangeDirectory(cdir);" << endl;
1185 out << " return kFALSE;" << endl;
1186 out << " }" << endl;
1187 out << " // Check for SETUP.C and execute" << endl;
1188 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1189 out << " printf(\"*******************************\\n\");" << endl;
1190 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1191 out << " printf(\"*******************************\\n\");" << endl;
1192 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1193 out << " } else {" << endl;
1194 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", package);" << endl;
1195 out << " gSystem->ChangeDirectory(cdir);" << endl;
1196 out << " return kFALSE;" << endl;
1197 out << " }" << endl;
1198 out << " // Restore original workdir" << endl;
1199 out << " gSystem->ChangeDirectory(cdir);" << endl;
1200 out << " return kTRUE;" << endl;
1201 out << "}" << endl;
1202 }
c57f56b7 1203 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1204 }
1205 Bool_t copy = kTRUE;
1206 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1207 if (copy) {
1208 CdWork();
1209 TString workdir = gGrid->GetHomeDirectory();
1210 workdir += fGridWorkingDir;
1211 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1212 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1213 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1214 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1215 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1216 }
1217 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1218 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1219 }
1220}
1221
1222//______________________________________________________________________________
1223void AliAnalysisAlien::WriteExecutable()
1224{
1225// Generate the alien executable script.
1226 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1227 ofstream out;
1228 out.open(fExecutable.Data(), ios::out);
1229 if (out.bad()) {
1230 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1231 return;
1232 }
1233 out << "#!/bin/bash" << endl;
1234 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1235 out << "echo \"=========================================\"" << endl;
1236 out << "echo \"############## PATH : ##############\"" << endl;
1237 out << "echo $PATH" << endl;
1238 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1239 out << "echo $LD_LIBRARY_PATH" << endl;
1240 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1241 out << "echo $ROOTSYS" << endl;
1242 out << "echo \"############## which root : ##############\"" << endl;
1243 out << "which root" << endl;
1244 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1245 out << "echo $ALICE_ROOT" << endl;
1246 out << "echo \"############## which aliroot : ##############\"" << endl;
1247 out << "which aliroot" << endl;
1248 out << "echo \"=========================================\"" << endl << endl;
1249// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1250 out << "root -b -q ";
1251 out << fAnalysisMacro.Data() << endl << endl;
1252 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1253 }
1254 Bool_t copy = kTRUE;
1255 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1256 if (copy) {
1257 CdWork();
1258 TString workdir = gGrid->GetHomeDirectory();
1259 workdir += fGridWorkingDir;
1260 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1261 if (FileExists(executable)) gGrid->Rm(executable);
1262 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1263 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1264 }
1265}
1266
1267//______________________________________________________________________________
1268void AliAnalysisAlien::WriteValidationScript()
1269{
1270// Generate the alien validation script.
1271 // Generate the validation script
1272 TObjString *os;
1273 if (!Connect()) {
1274 Error("WriteValidationScript", "Alien connection required");
1275 return;
1276 }
1277 TString out_stream = "";
1278 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1279 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1280 ofstream out;
1281 out.open("validate.sh", ios::out);
1282 out << "#!/bin/bash" << endl;
1283 out << "##################################################" << endl;
1284 out << "validateout=`dirname $0`" << endl;
1285 out << "validatetime=`date`" << endl;
1286 out << "validated=\"0\";" << endl;
1287 out << "error=0" << endl;
1288 out << "if [ -z $validateout ]" << endl;
1289 out << "then" << endl;
1290 out << " validateout=\".\"" << endl;
1291 out << "fi" << endl << endl;
1292 out << "cd $validateout;" << endl;
1293 out << "validateworkdir=`pwd`;" << endl << endl;
1294 out << "echo \"*******************************************************\"" << out_stream << endl;
1295 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1296 out << "" << endl;
1297 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1298 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1299 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1300 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1301 out << "ls -la ./" << out_stream << endl;
1302 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1303 out << "##################################################" << endl;
1304 TObjArray *arr = fOutputFiles.Tokenize(" ");
1305 TIter next1(arr);
1306 TString output_file;
1307 while ((os=(TObjString*)next1())) {
1308 output_file = os->GetString();
1309 Int_t index = output_file.Index("@");
1310 if (index > 0) output_file.Remove(index);
1311 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1312 out << " error=1" << endl;
1313 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1314 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1315 out << "fi" << endl;
1316 }
1317 delete arr;
1318 out << "if [ $error = 0 ] ; then" << endl;
1319 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1320 out << "fi" << endl;
1321
1322 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1323 out << "echo \"*******************************************************\"" << out_stream << endl;
1324 out << "cd -" << endl;
1325 out << "exit $error" << endl;
1326 }
1327 Bool_t copy = kTRUE;
1328 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1329 if (copy) {
1330 CdWork();
1331 TString workdir = gGrid->GetHomeDirectory();
1332 workdir += fGridWorkingDir;
1333 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1334 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1335 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1336 }
1337}