]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
new round of documentation
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TFileMerger.h"
34#include "AliAnalysisManager.h"
bb885a9e 35#include "AliVEventHandler.h"
36#include "AliAnalysisDataContainer.h"
c57f56b7 37#include "AliAnalysisAlien.h"
38
39ClassImp(AliAnalysisAlien)
40
41//______________________________________________________________________________
42AliAnalysisAlien::AliAnalysisAlien()
43 :AliAnalysisGrid(),
44 fGridJDL(NULL),
45 fPrice(0),
46 fTTL(0),
47 fSplitMaxInputFileNumber(0),
48 fMaxInitFailed(0),
49 fMasterResubmitThreshold(0),
bb885a9e 50 fNtestFiles(0),
a8739e8a 51 fNMasterJobs(0),
16a4353c 52 fMaxMergeFiles(0),
c57f56b7 53 fRunNumbers(),
54 fExecutable(),
55 fArguments(),
56 fAnalysisMacro(),
57 fAnalysisSource(),
58 fAdditionalLibs(),
59 fSplitMode(),
60 fAPIVersion(),
61 fROOTVersion(),
62 fAliROOTVersion(),
63 fUser(),
64 fGridWorkingDir(),
65 fGridDataDir(),
66 fDataPattern(),
67 fGridOutputDir(),
68 fOutputArchive(),
69 fOutputFiles(),
70 fInputFormat(),
e7c71df0 71 fDatasetName(),
c57f56b7 72 fJDLName(),
bb885a9e 73 fMergeExcludes(),
f965131e 74 fIncludePath(),
bb885a9e 75 fCloseSE(),
0df6ccf2 76 fFriendChainName(),
4e5c5506 77 fInputFiles(0),
78 fPackages(0)
c57f56b7 79{
80// Dummy ctor.
81 SetDefaults();
82}
83
84//______________________________________________________________________________
85AliAnalysisAlien::AliAnalysisAlien(const char *name)
86 :AliAnalysisGrid(name),
87 fGridJDL(NULL),
88 fPrice(0),
89 fTTL(0),
90 fSplitMaxInputFileNumber(0),
91 fMaxInitFailed(0),
92 fMasterResubmitThreshold(0),
bb885a9e 93 fNtestFiles(0),
a8739e8a 94 fNMasterJobs(0),
16a4353c 95 fMaxMergeFiles(0),
c57f56b7 96 fRunNumbers(),
97 fExecutable(),
98 fArguments(),
99 fAnalysisMacro(),
100 fAnalysisSource(),
101 fAdditionalLibs(),
102 fSplitMode(),
103 fAPIVersion(),
104 fROOTVersion(),
105 fAliROOTVersion(),
106 fUser(),
107 fGridWorkingDir(),
108 fGridDataDir(),
109 fDataPattern(),
110 fGridOutputDir(),
111 fOutputArchive(),
112 fOutputFiles(),
113 fInputFormat(),
e7c71df0 114 fDatasetName(),
c57f56b7 115 fJDLName(),
bb885a9e 116 fMergeExcludes(),
f965131e 117 fIncludePath(),
bb885a9e 118 fCloseSE(),
0df6ccf2 119 fFriendChainName(),
4e5c5506 120 fInputFiles(0),
121 fPackages(0)
c57f56b7 122{
123// Default ctor.
124 SetDefaults();
125}
126
127//______________________________________________________________________________
128AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
129 :AliAnalysisGrid(other),
130 fGridJDL(NULL),
131 fPrice(other.fPrice),
132 fTTL(other.fTTL),
133 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
134 fMaxInitFailed(other.fMaxInitFailed),
135 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 136 fNtestFiles(other.fNtestFiles),
a8739e8a 137 fNMasterJobs(other.fNMasterJobs),
16a4353c 138 fMaxMergeFiles(other.fMaxMergeFiles),
c57f56b7 139 fRunNumbers(other.fRunNumbers),
140 fExecutable(other.fExecutable),
141 fArguments(other.fArguments),
142 fAnalysisMacro(other.fAnalysisMacro),
143 fAnalysisSource(other.fAnalysisSource),
144 fAdditionalLibs(other.fAdditionalLibs),
145 fSplitMode(other.fSplitMode),
146 fAPIVersion(other.fAPIVersion),
147 fROOTVersion(other.fROOTVersion),
148 fAliROOTVersion(other.fAliROOTVersion),
149 fUser(other.fUser),
150 fGridWorkingDir(other.fGridWorkingDir),
151 fGridDataDir(other.fGridDataDir),
152 fDataPattern(other.fDataPattern),
153 fGridOutputDir(other.fGridOutputDir),
154 fOutputArchive(other.fOutputArchive),
155 fOutputFiles(other.fOutputFiles),
156 fInputFormat(other.fInputFormat),
e7c71df0 157 fDatasetName(other.fDatasetName),
c57f56b7 158 fJDLName(other.fJDLName),
bb885a9e 159 fMergeExcludes(other.fMergeExcludes),
f965131e 160 fIncludePath(other.fIncludePath),
bb885a9e 161 fCloseSE(other.fCloseSE),
0df6ccf2 162 fFriendChainName(other.fFriendChainName),
4e5c5506 163 fInputFiles(0),
164 fPackages(0)
c57f56b7 165{
166// Copy ctor.
167 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 168 fRunRange[0] = other.fRunRange[0];
169 fRunRange[1] = other.fRunRange[1];
c57f56b7 170 if (other.fInputFiles) {
171 fInputFiles = new TObjArray();
172 TIter next(other.fInputFiles);
173 TObject *obj;
174 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
175 fInputFiles->SetOwner();
176 }
4e5c5506 177 if (other.fPackages) {
178 fPackages = new TObjArray();
179 TIter next(other.fPackages);
180 TObject *obj;
181 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
182 fPackages->SetOwner();
183 }
c57f56b7 184}
185
186//______________________________________________________________________________
187AliAnalysisAlien::~AliAnalysisAlien()
188{
189// Destructor.
190 if (fGridJDL) delete fGridJDL;
191 if (fInputFiles) delete fInputFiles;
4e5c5506 192 if (fPackages) delete fPackages;
c57f56b7 193}
194
195//______________________________________________________________________________
196AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
197{
198// Assignment.
199 if (this != &other) {
200 AliAnalysisGrid::operator=(other);
201 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
202 fPrice = other.fPrice;
203 fTTL = other.fTTL;
204 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
205 fMaxInitFailed = other.fMaxInitFailed;
206 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 207 fNtestFiles = other.fNtestFiles;
c57f56b7 208 fRunNumbers = other.fRunNumbers;
209 fExecutable = other.fExecutable;
210 fArguments = other.fArguments;
211 fAnalysisMacro = other.fAnalysisMacro;
212 fAnalysisSource = other.fAnalysisSource;
213 fAdditionalLibs = other.fAdditionalLibs;
214 fSplitMode = other.fSplitMode;
215 fAPIVersion = other.fAPIVersion;
216 fROOTVersion = other.fROOTVersion;
217 fAliROOTVersion = other.fAliROOTVersion;
218 fUser = other.fUser;
219 fGridWorkingDir = other.fGridWorkingDir;
220 fGridDataDir = other.fGridDataDir;
221 fDataPattern = other.fDataPattern;
222 fGridOutputDir = other.fGridOutputDir;
223 fOutputArchive = other.fOutputArchive;
224 fOutputFiles = other.fOutputFiles;
225 fInputFormat = other.fInputFormat;
e7c71df0 226 fDatasetName = other.fDatasetName;
c57f56b7 227 fJDLName = other.fJDLName;
bb885a9e 228 fMergeExcludes = other.fMergeExcludes;
f965131e 229 fIncludePath = other.fIncludePath;
bb885a9e 230 fCloseSE = other.fCloseSE;
0df6ccf2 231 fFriendChainName = other.fFriendChainName;
c57f56b7 232 if (other.fInputFiles) {
233 fInputFiles = new TObjArray();
234 TIter next(other.fInputFiles);
235 TObject *obj;
236 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
237 fInputFiles->SetOwner();
238 }
4e5c5506 239 if (other.fPackages) {
240 fPackages = new TObjArray();
241 TIter next(other.fPackages);
242 TObject *obj;
243 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
244 fPackages->SetOwner();
245 }
c57f56b7 246 }
247 return *this;
248}
249
f965131e 250//______________________________________________________________________________
251void AliAnalysisAlien::AddIncludePath(const char *path)
252{
253// Add include path in the remote analysis macro.
254 TString p(path);
255 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
256 else fIncludePath += Form("-I%s ", path);
257}
258
c57f56b7 259//______________________________________________________________________________
260void AliAnalysisAlien::AddRunNumber(Int_t run)
261{
262// Add a run number to the list of runs to be processed.
263 if (fRunNumbers.Length()) fRunNumbers += " ";
264 fRunNumbers += Form("%d", run);
265}
266
267//______________________________________________________________________________
268void AliAnalysisAlien::AddDataFile(const char *lfn)
269{
270// Adds a data file to the input to be analysed. The file should be a valid LFN
271// or point to an existing file in the alien workdir.
272 if (!fInputFiles) fInputFiles = new TObjArray();
273 fInputFiles->Add(new TObjString(lfn));
274}
275
276//______________________________________________________________________________
277Bool_t AliAnalysisAlien::Connect()
278{
279// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
280 if (gGrid && gGrid->IsConnected()) return kTRUE;
281 if (!gSystem->Getenv("alien_API_USER")) {
282 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
283 gSystem->Getenv("UID"));
284 return kFALSE;
285 }
286 if (!gGrid) {
287 Info("Connect", "Trying to connect to AliEn ...");
288 TGrid::Connect("alien://");
289 }
290 if (!gGrid || !gGrid->IsConnected()) {
291 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
292 return kFALSE;
293 }
294 fUser = gGrid->GetUser();
295 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
296 return kTRUE;
297}
298
299//______________________________________________________________________________
300void AliAnalysisAlien::CdWork()
301{
302// Check validity of alien workspace. Create directory if possible.
303 if (!Connect()) {
304 Error("CdWork", "Alien connection required");
305 return;
306 }
307 TString homedir = gGrid->GetHomeDirectory();
308 TString workdir = homedir + fGridWorkingDir;
309 if (!gGrid->Cd(workdir)) {
310 gGrid->Cd(homedir);
311 if (gGrid->Mkdir(workdir)) {
312 gGrid->Cd(fGridWorkingDir);
313 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
314 } else {
315 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
316 workdir.Data(), homedir.Data());
317 fGridWorkingDir = "";
318 }
319 }
320}
321
322//______________________________________________________________________________
323Bool_t AliAnalysisAlien::CheckInputData()
324{
325// Check validity of input data. If necessary, create xml files.
a8739e8a 326 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
327 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile()/SetRunRange().");
c57f56b7 328 return kFALSE;
329 }
330 // Process declared files
331 Bool_t is_collection = kFALSE;
332 Bool_t is_xml = kFALSE;
333 Bool_t use_tags = kFALSE;
334 Bool_t checked = kFALSE;
335 CdWork();
336 TString file;
337 TString workdir = gGrid->GetHomeDirectory();
338 workdir += fGridWorkingDir;
339 if (fInputFiles) {
340 TObjString *objstr;
341 TIter next(fInputFiles);
342 while ((objstr=(TObjString*)next())) {
343 file = workdir;
344 file += "/";
345 file += objstr->GetString();
346 // Store full lfn path
347 if (FileExists(file)) objstr->SetString(file);
348 else {
349 file = objstr->GetName();
350 if (!FileExists(objstr->GetName())) {
351 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
352 objstr->GetName(), workdir.Data());
353 return kFALSE;
354 }
355 }
356 Bool_t iscoll, isxml, usetags;
357 CheckDataType(file, iscoll, isxml, usetags);
358 if (!checked) {
359 checked = kTRUE;
360 is_collection = iscoll;
361 is_xml = isxml;
362 use_tags = usetags;
363 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
364 } else {
365 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
366 Error("CheckInputData", "Some conflict was found in the types of inputs");
367 return kFALSE;
368 }
369 }
370 }
371 }
372 // Process requested run numbers
a8739e8a 373 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 374 // Check validity of alien data directory
375 if (!fGridDataDir.Length()) {
376 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
377 return kFALSE;
378 }
379 if (!gGrid->Cd(fGridDataDir)) {
380 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
381 return kFALSE;
382 }
383 if (is_collection) {
384 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
385 return kFALSE;
386 }
387
388 if (checked && !is_xml) {
389 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
390 return kFALSE;
391 }
392 // Check validity of run number(s)
393 TObjArray *arr;
394 TObjString *os;
395 TString path;
396 if (!checked) {
397 checked = kTRUE;
398 use_tags = fDataPattern.Contains("tag");
399 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
400 }
401 if (use_tags != fDataPattern.Contains("tag")) {
402 Error("CheckInputData", "Cannot mix input files using/not using tags");
403 return kFALSE;
404 }
405 if (fRunNumbers.Length()) {
a8739e8a 406 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 407 arr = fRunNumbers.Tokenize(" ");
408 TIter next(arr);
409 while ((os=(TObjString*)next())) {
410 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
411 if (!gGrid->Cd(path)) {
a8739e8a 412 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
413 continue;
c57f56b7 414 }
415 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
416 TString msg = "\n##### file: ";
417 msg += path;
418 msg += " type: xml_collection;";
419 if (use_tags) msg += " using_tags: Yes";
420 else msg += " using_tags: No";
421 Info("CheckDataType", msg.Data());
422 AddDataFile(path);
423 }
424 delete arr;
a8739e8a 425 } else {
426 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
427 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
428 path = Form("%s/%d ", fGridDataDir.Data(), irun);
429 if (!gGrid->Cd(path)) {
430 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
431 continue;
432 }
433 path = Form("%s/%d.xml", workdir.Data(),irun);
434 TString msg = "\n##### file: ";
435 msg += path;
436 msg += " type: xml_collection;";
437 if (use_tags) msg += " using_tags: Yes";
438 else msg += " using_tags: No";
439 Info("CheckDataType", msg.Data());
440 AddDataFile(path);
441 }
c57f56b7 442 }
443 return kTRUE;
444}
445
446//______________________________________________________________________________
447Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
448{
449// Create dataset for the grid data directory + run number.
450 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
451 if (!Connect()) {
452 Error("CreateDataset", "Cannot create dataset with no grid connection");
453 return kFALSE;
454 }
455
456 // Cd workspace
457 CdWork();
458 TString workdir = gGrid->GetHomeDirectory();
459 workdir += fGridWorkingDir;
460
461 // Compose the 'find' command arguments
462 TString command;
463 TString options = "-x collection ";
bb885a9e 464 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 465 TString conditions = "";
466
467 TString file;
468 TString path;
a8739e8a 469 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 470 // Several runs
a8739e8a 471 if (fRunNumbers.Length()) {
472 TObjArray *arr = fRunNumbers.Tokenize(" ");
473 TObjString *os;
474 TIter next(arr);
475 while ((os=(TObjString*)next())) {
476 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
477 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
478 else file = Form("%s.xml", os->GetString().Data());
479 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
480 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
481// gGrid->Rm(file);
482 continue;
483 }
484 command = "find ";
485 command += options;
486 command += path;
487 command += pattern;
488// conditions = Form(" > %s", file.Data());
489 command += conditions;
490 TGridResult *res = gGrid->Command(command);
491 if (res) delete res;
492 // Write standard output to file
493 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
494 if (TestBit(AliAnalysisGrid::kTest)) break;
495 // Copy xml file to alien space
496 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
497 if (!FileExists(file)) {
498 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
499 delete arr;
500 return kFALSE;
501 }
502 }
503 delete arr;
504 } else {
505 // Process a full run range.
506 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
507 path = Form("%s/%d ", fGridDataDir.Data(), irun);
508 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
509 else file = Form("%d.xml", irun);
510 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
511 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
512// gGrid->Rm(file);
513 continue;
514 }
515 command = "find ";
516 command += options;
517 command += path;
518 command += pattern;
519// conditions = Form(" > %s", file.Data());
520 command += conditions;
521 TGridResult *res = gGrid->Command(command);
522 if (res) delete res;
523 // Write standard output to file
524 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
525 if (TestBit(AliAnalysisGrid::kTest)) break;
526 // Copy xml file to alien space
527 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
528 if (!FileExists(file)) {
529 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
530 return kFALSE;
531 }
c57f56b7 532 }
a8739e8a 533 }
c57f56b7 534 return kTRUE;
535}
536
537//______________________________________________________________________________
538Bool_t AliAnalysisAlien::CreateJDL()
539{
540// Generate a JDL file according to current settings. The name of the file is
541// specified by fJDLName.
542 Bool_t error = kFALSE;
543 TObjArray *arr = 0;
544 Bool_t copy = kTRUE;
545 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
546 Bool_t generate = kTRUE;
547 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
548 if (!Connect()) {
549 Error("CreateJDL", "Alien connection required");
550 return kFALSE;
551 }
552 // Check validity of alien workspace
553 CdWork();
554 TString workdir = gGrid->GetHomeDirectory();
555 workdir += fGridWorkingDir;
556 if (generate) {
557 TObjString *os;
558 if (!fInputFiles) {
559 Error("CreateJDL()", "Define some input files for your analysis.");
560 error = kTRUE;
561 }
562 // Compose list of input files
563 // Check if output files were defined
564 if (!fOutputFiles.Length()) {
565 Error("CreateJDL", "You must define at least one output file");
566 error = kTRUE;
567 }
568 // Check if an output directory was defined and valid
569 if (!fGridOutputDir.Length()) {
570 Error("CreateJDL", "You must define AliEn output directory");
571 error = kTRUE;
572 } else {
573 if (!gGrid->Cd(fGridOutputDir)) {
574 if (gGrid->Mkdir(fGridOutputDir)) {
575 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
576 } else {
577 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
578 error = kTRUE;
579 }
580 }
581 gGrid->Cd(workdir);
582 }
583 // Exit if any error up to now
584 if (error) return kFALSE;
585 // Set JDL fields
586 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
587 fGridJDL->SetExecutable(fExecutable);
588// fGridJDL->SetTTL((UInt_t)fTTL);
589 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
590 if (fMaxInitFailed > 0)
591 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
592 if (fSplitMaxInputFileNumber > 0)
593 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
594 if (fSplitMode.Length())
595 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
596// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
597 if (fAliROOTVersion.Length())
598 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
599 if (fROOTVersion.Length())
600 fGridJDL->AddToPackages("ROOT", fROOTVersion);
601 if (fAPIVersion.Length())
602 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
603 fGridJDL->SetInputDataListFormat(fInputFormat);
604 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 605 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
606 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
607 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
608 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
609 if (fAdditionalLibs.Length()) {
610 arr = fAdditionalLibs.Tokenize(" ");
611 TIter next(arr);
612 while ((os=(TObjString*)next())) {
613 if (os->GetString().Contains(".so")) continue;
614 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
615 }
616 delete arr;
617 }
4e5c5506 618 if (fPackages) {
619 TIter next(fPackages);
620 TObject *obj;
621 while ((obj=next()))
622 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
623 }
c57f56b7 624 if (fOutputArchive.Length()) {
625 arr = fOutputArchive.Tokenize(" ");
626 TIter next(arr);
627 while ((os=(TObjString*)next()))
43da816a 628 if (!os->GetString().Contains("@") && fCloseSE.Length())
629 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
630 else
631 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 632 delete arr;
633 }
c57f56b7 634 arr = fOutputFiles.Tokenize(" ");
635 TIter next(arr);
43da816a 636 while ((os=(TObjString*)next())) {
637 if (!os->GetString().Contains("@") && fCloseSE.Length())
638 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
639 else
640 fGridJDL->AddToOutputSandbox(os->GetString());
641 }
c57f56b7 642 delete arr;
643// fGridJDL->SetPrice((UInt_t)fPrice);
644 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
645 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
646 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
a8739e8a 647 // Depending if going through a run range or not, generate one or mode jdl's
648 if (!fRunRange[0]) WriteJDL(-1,copy);
649 else {
650 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++)
b15e08c1 651 if (!WriteJDL(irun-fRunRange[0],copy)) break;
c57f56b7 652 }
c57f56b7 653 }
654 // Copy jdl to grid workspace
a8739e8a 655 if (copy) {
c57f56b7 656 if (fAdditionalLibs.Length()) {
e7c71df0 657 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 658 TObjString *os;
659 TIter next(arr);
660 while ((os=(TObjString*)next())) {
c57f56b7 661 if (os->GetString().Contains(".so")) continue;
4e5c5506 662 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 663 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
664 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
665 }
666 delete arr;
667 }
4e5c5506 668 if (fPackages) {
669 TIter next(fPackages);
670 TObject *obj;
671 while ((obj=next())) {
672 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
673 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
674 }
675 }
c57f56b7 676 }
677 return kTRUE;
678}
679
a8739e8a 680//______________________________________________________________________________
681Bool_t AliAnalysisAlien::WriteJDL(Int_t findex, Bool_t copy)
682{
683// Writes one or more JDL's corresponding to findex. If findex is negative,
684// all run numbers are considered in one go (jdl). For non-negative indices
685// they correspond to the indices in the array fInputFiles.
686 if (!fInputFiles) return kFALSE;
687 TObjString *os;
688 TString line;
689 TString workdir = gGrid->GetHomeDirectory();
690 workdir += fGridWorkingDir;
a8739e8a 691 if (findex < 0) {
692 TIter next(fInputFiles);
693 while ((os=(TObjString*)next()))
694 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
695 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
696 } else {
697 os = (TObjString*)fInputFiles->At(findex);
b15e08c1 698 if (!os) return kFALSE;
a8739e8a 699 line = "#Input xml collection\n";
700 line += "InputDataCollection = {";
701 line += Form(" \"LF:%s,nodownload\"", os->GetString().Data());
702 line += "\n};\n";
703 fGridJDL->SetOutputDirectory(Form("%s/%s/%03d#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data(),findex));
704 }
705
706
707 // Generate the JDL as a string
708 TString sjdl = fGridJDL->Generate();
709 Int_t index;
710 index = sjdl.Index("Executable");
711 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
712 index = sjdl.Index("Split ");
713 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
714 index = sjdl.Index("SplitMaxInputFileNumber");
715 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
716 index = sjdl.Index("InputDataCollection");
717 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
718 else {
719 index = sjdl.Index("OutputDir");
720 sjdl.Insert(index, line.Data());
721 }
722 index = sjdl.Index("InputFile");
723 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
724 index = sjdl.Index("InputDataList ");
725 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
726 index = sjdl.Index("InputDataListFormat");
727 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
728 index = sjdl.Index("Price");
729 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
730 index = sjdl.Index("Requirements");
731 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
732 index = sjdl.Index("Packages");
733 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
734 index = sjdl.Index("User =");
735 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
736 index = sjdl.Index("TTL");
737 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
738 index = sjdl.Index("OutputFile");
739 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
740 index = sjdl.Index("OutputDir");
741 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
742 index = sjdl.Index("OutputArchive");
743 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
744 index = sjdl.Index("MaxInitFailed");
745 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
746 index = sjdl.Index("MasterResubmitThreshold");
747 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
748 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
749 index = sjdl.Index("Validationcommand");
750 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
751 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
752 sjdl.ReplaceAll("(member", "\n (member");
753 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
754 sjdl.ReplaceAll("{", "{\n ");
755 sjdl.ReplaceAll("};", "\n};");
756 sjdl.ReplaceAll("{\n \n", "{\n");
757 sjdl.ReplaceAll("\n\n", "\n");
758 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
759 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
760 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
761 index = sjdl.Index("JDLVariables");
762 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
763 // Write jdl to file
764 TString snjdl = fJDLName;
765 if (findex >=0) snjdl.ReplaceAll(".jdl", Form("_%03d.jdl", findex));
766 ofstream out;
767 out.open(snjdl.Data(), ios::out);
768 if (out.bad()) {
769 Error("CreateJDL", "Bad file name: %s", snjdl.Data());
770 return kFALSE;
771 }
772 out << sjdl << endl;
773
774 // Copy jdl to grid workspace
775 if (!copy) {
776 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
777 } else {
778 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", snjdl.Data());
779 if (FileExists(snjdl)) gGrid->Rm(snjdl);
780 TFile::Cp(Form("file:%s",snjdl.Data()), Form("alien://%s/%s", workdir.Data(), snjdl.Data()));
781 }
782 return kTRUE;
783}
784
c57f56b7 785//______________________________________________________________________________
786Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
787{
788// Returns true if file exists.
789 if (!gGrid) {
790 Error("FileExists", "No connection to grid");
791 return kFALSE;
792 }
793 TGridResult *res = gGrid->Ls(lfn);
794 if (!res) return kFALSE;
795 TMap *map = dynamic_cast<TMap*>(res->At(0));
796 if (!map) {
797 delete res;
798 return kFALSE;
799 }
800 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
801 if (!objs || !objs->GetString().Length()) {
802 delete res;
803 return kFALSE;
804 }
805 delete res;
806 return kTRUE;
807}
808
809//______________________________________________________________________________
810void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
811{
812// Check input data type.
813 is_collection = kFALSE;
814 is_xml = kFALSE;
815 use_tags = kFALSE;
816 if (!gGrid) {
817 Error("CheckDataType", "No connection to grid");
818 return;
819 }
820 is_collection = IsCollection(lfn);
821 TString msg = "\n##### file: ";
822 msg += lfn;
823 if (is_collection) {
824 msg += " type: raw_collection;";
825 // special treatment for collections
826 is_xml = kFALSE;
827 // check for tag files in the collection
828 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
829 if (!res) {
830 msg += " using_tags: No (unknown)";
831 Info("CheckDataType", msg.Data());
832 return;
833 }
834 const char* typeStr = res->GetKey(0, "origLFN");
835 if (!typeStr || !strlen(typeStr)) {
836 msg += " using_tags: No (unknown)";
837 Info("CheckDataType", msg.Data());
838 return;
839 }
840 TString file = typeStr;
841 use_tags = file.Contains(".tag");
842 if (use_tags) msg += " using_tags: Yes";
843 else msg += " using_tags: No";
844 Info("CheckDataType", msg.Data());
845 return;
846 }
847 TString slfn(lfn);
848 slfn.ToLower();
849 is_xml = slfn.Contains(".xml");
850 if (is_xml) {
851 // Open xml collection and check if there are tag files inside
852 msg += " type: xml_collection;";
853 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
854 if (!coll) {
855 msg += " using_tags: No (unknown)";
856 Info("CheckDataType", msg.Data());
857 return;
858 }
859 TMap *map = coll->Next();
860 if (!map) {
861 msg += " using_tags: No (unknown)";
862 Info("CheckDataType", msg.Data());
863 return;
864 }
865 map = (TMap*)map->GetValue("");
866 TString file;
867 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
868 use_tags = file.Contains(".tag");
869 delete coll;
870 if (use_tags) msg += " using_tags: Yes";
871 else msg += " using_tags: No";
872 Info("CheckDataType", msg.Data());
873 return;
874 }
875 use_tags = slfn.Contains(".tag");
876 if (slfn.Contains(".root")) msg += " type: root file;";
877 else msg += " type: unhnown file;";
878 if (use_tags) msg += " using_tags: Yes";
879 else msg += " using_tags: No";
880 Info("CheckDataType", msg.Data());
881}
882
4e5c5506 883//______________________________________________________________________________
884void AliAnalysisAlien::EnablePackage(const char *package)
885{
886// Enables a par file supposed to exist in the current directory.
887 TString pkg(package);
888 pkg.ReplaceAll(".par", "");
889 pkg += ".par";
890 if (gSystem->AccessPathName(pkg)) {
891 Error("EnablePackage", "Package %s not found", pkg.Data());
892 return;
893 }
894 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
895 Info("EnablePackage", "AliEn plugin will use .par packages");
896 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
897 if (!fPackages) {
898 fPackages = new TObjArray();
899 fPackages->SetOwner();
900 }
901 fPackages->Add(new TObjString(pkg));
902}
903
c57f56b7 904//______________________________________________________________________________
905Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
906{
907// Returns true if file is a collection. Functionality duplicated from
908// TAlien::Type() because we don't want to directly depend on TAlien.
909 if (!gGrid) {
910 Error("IsCollection", "No connection to grid");
911 return kFALSE;
912 }
913 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
914 if (!res) return kFALSE;
915 const char* typeStr = res->GetKey(0, "type");
916 if (!typeStr || !strlen(typeStr)) return kFALSE;
917 if (!strcmp(typeStr, "collection")) return kTRUE;
918 delete res;
919 return kFALSE;
920}
921
16a4353c 922//______________________________________________________________________________
923void AliAnalysisAlien::Print(Option_t *) const
924{
925// Print current plugin settings.
926 printf("### AliEn analysis plugin current settings ###\n");
927 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
928 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
929 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
930 if (fUser.Length())
931 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
932 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
933 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
934 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
935 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
936 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
937 if (fRunNumbers.Length())
938 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
939 if (fRunRange[0])
940 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
941 if (!fRunRange[0] && !fRunNumbers.Length()) {
942 TIter next(fInputFiles);
943 TObject *obj;
944 TString list;
945 while ((obj=next())) list += obj->GetName();
946 printf("= Input files to be processed: _________________ %s\n", list.Data());
947 }
948 if (TestBit(AliAnalysisGrid::kTest))
949 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
950 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
951 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
952 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
953 printf("=====================================================================\n");
954 printf("= Job price: ___________________________________ %d\n", fPrice);
955 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
956 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
957 if (fMaxInitFailed>0)
958 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
959 if (fMasterResubmitThreshold>0)
960 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
961 if (fNMasterJobs>0)
962 printf("= Number of master jobs to be launched: ________ not used yet\n");
963 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
964 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
965 if (fArguments.Length())
966 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
967 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
968 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
969 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
970 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
971 if (fDatasetName)
972 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
973 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
974 if (fIncludePath.Data())
975 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
976 if (fCloseSE.Length())
977 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
978 if (fFriendChainName.Length())
979 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
980 if (fPackages) {
981 TIter next(fPackages);
982 TObject *obj;
983 TString list;
984 while ((obj=next())) list += obj->GetName();
985 printf("= Par files to be used: ________________________ %s\n", list.Data());
986 }
987}
988
c57f56b7 989//______________________________________________________________________________
990void AliAnalysisAlien::SetDefaults()
991{
992// Set default values for everything. What cannot be filled will be left empty.
993 if (fGridJDL) delete fGridJDL;
994 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
995 fPrice = 1;
996 fTTL = 30000;
997 fSplitMaxInputFileNumber = 100;
998 fMaxInitFailed = 0;
999 fMasterResubmitThreshold = 0;
bb885a9e 1000 fNtestFiles = 10;
a8739e8a 1001 fRunRange[0] = 0;
1002 fRunRange[1] = 0;
1003 fNMasterJobs = 0;
16a4353c 1004 fMaxMergeFiles = 100;
c57f56b7 1005 fRunNumbers = "";
1006 fExecutable = "analysis.sh";
1007 fArguments = "";
1008 fAnalysisMacro = "myAnalysis.C";
1009 fAnalysisSource = "";
1010 fAdditionalLibs = "";
1011 fSplitMode = "se";
1012 fAPIVersion = "";
1013 fROOTVersion = "";
1014 fAliROOTVersion = "";
1015 fUser = ""; // Your alien user name
1016 fGridWorkingDir = "";
1017 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1018 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1019 fFriendChainName = "";
c57f56b7 1020 fGridOutputDir = "output";
1021 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1022 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1023 fInputFormat = "xml-single";
1024 fJDLName = "analysis.jdl";
bb885a9e 1025 fMergeExcludes = "";
c57f56b7 1026}
1027
1028//______________________________________________________________________________
1029Bool_t AliAnalysisAlien::MergeOutputs()
1030{
1031// Merge analysis outputs existing in the AliEn space.
1032 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1033 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1034 if (!Connect()) {
1035 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1036 return kFALSE;
1037 }
1038 // Get the output path
1039 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1040 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
1041 if (!gGrid->Cd(output)) {
1042 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1043 return kFALSE;
1044 }
1045 if (!fOutputFiles.Length()) {
1046 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1047 return kFALSE;
1048 }
1049 TObjArray *list = fOutputFiles.Tokenize(" ");
1050 TIter next(list);
1051 TObjString *str;
1052 TString command;
1053 TString output_file;
16a4353c 1054 TString output_chunk;
1055 TString previous_chunk;
1056 Int_t count_chunk = 0;
1057 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1058 Bool_t merged = kTRUE;
1059 while((str=(TObjString*)next())) {
1060 output_file = str->GetString();
1061 Int_t index = output_file.Index("@");
1062 if (index > 0) output_file.Remove(index);
bb885a9e 1063 if (fMergeExcludes.Length() &&
1064 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1065 // Perform a 'find' command in the output directory, looking for registered outputs
c57f56b7 1066 command = Form("find %s/ *%s", output.Data(), output_file.Data());
1067 printf("command: %s\n", command.Data());
1068 TGridResult *res = gGrid->Command(command);
1069 if (!res) continue;
1070 TFileMerger *fm = 0;
1071 TIter nextmap(res);
1072 TMap *map;
16a4353c 1073 previous_chunk = "";
1074 count_chunk = 0;
1075 count_zero = fMaxMergeFiles;
c57f56b7 1076 while ((map=(TMap*)nextmap())) {
16a4353c 1077 // Loop 'find' results and get next LFN
1078 if (count_zero == fMaxMergeFiles) {
1079 // First file in chunk - create file merger and add previous chunk if any.
1080 fm = new TFileMerger(kFALSE);
1081 fm->SetFastMethod(kTRUE);
1082 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1083 output_chunk = output_file;
1084 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1085 }
1086 // If last file found, put merged results in the output file
1087 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1088 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1089 if (!objs || !objs->GetString().Length()) {
16a4353c 1090 // Nothing found - skip this output
c57f56b7 1091 delete res;
16a4353c 1092 delete fm;
1093 break;
1094 }
1095 // Add file to be merged and decrement chunk counter.
1096 fm->AddFile(objs->GetString());
1097 count_zero--;
1098 if (count_zero==0 || map == res->Last()) {
1099 fm->OutputFile(output_chunk);
1100 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1101 // Nothing found - skip this output
1102 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1103 delete res;
1104 delete fm;
1105 break;
1106 }
1107 // Merge the outputs, then go to next chunk
1108 if (!fm->Merge()) {
1109 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1110 delete res;
1111 delete fm;
1112 merged = kFALSE;
1113 break;
1114 } else {
1115 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
1116 }
1117 if (map == res->Last()) {
1118 delete res;
1119 delete fm;
1120 break;
1121 }
1122 count_chunk++;
1123 count_zero = fMaxMergeFiles;
1124 previous_chunk = output_chunk;
c57f56b7 1125 }
c57f56b7 1126 }
c57f56b7 1127 }
1128 if (!merged) {
1129 Error("MergeOutputs", "Terminate() will NOT be executed");
1130 }
1131 return merged;
1132}
1133
bb885a9e 1134//______________________________________________________________________________
1135void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1136{
1137// Use the output files connected to output containers from the analysis manager
1138// rather than the files defined by SetOutputFiles
1139 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1140 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1141 analysis manager");
1142 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1143}
1144
c57f56b7 1145//______________________________________________________________________________
1146void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
1147{
1148// Start remote grid analysis.
1149
43da816a 1150 // Check if output files have to be taken from the analysis manager
1151 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1152 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1153 if (!mgr || !mgr->IsInitialized()) {
1154 Error("StartAnalysis", "You need an initialized analysis manager for this");
1155 return;
1156 }
1157 fOutputFiles = "";
1158 TIter next(mgr->GetOutputs());
1159 AliAnalysisDataContainer *output;
1160 while ((output=(AliAnalysisDataContainer*)next())) {
1161 const char *filename = output->GetFileName();
1162 if (!(strcmp(filename, "default"))) {
1163 if (!mgr->GetOutputEventHandler()) continue;
1164 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1165 }
1166 if (fOutputFiles.Length()) fOutputFiles += " ";
1167 fOutputFiles += filename;
1168 }
c07b9ce2 1169 // Add extra files registered to the analysis manager
1170 if (mgr->GetExtraFiles().Length()) {
1171 if (fOutputFiles.Length()) fOutputFiles += " ";
1172 fOutputFiles += mgr->GetExtraFiles();
1173 }
43da816a 1174 }
f7b1cbc2 1175// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1176 if (TestBit(AliAnalysisGrid::kOffline)) {
1177 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1178 \n there nor any job run. You can revise the JDL and analysis \
1179 \n macro then run the same in \"submit\" mode.");
1180 } else if (TestBit(AliAnalysisGrid::kTest)) {
1181 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1182 \n dataset.");
1183 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1184 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1185 \n space and job submitted.");
1186 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1187 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
1188 return;
1189 } else {
1190 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1191 }
1192
1193 if (!Connect()) {
1194 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
1195 return;
16a4353c 1196 }
1197 Print();
c57f56b7 1198 if (!CheckInputData()) {
1199 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
1200 return;
1201 }
1202 CreateDataset(fDataPattern);
1203 WriteAnalysisFile();
1204 WriteAnalysisMacro();
1205 WriteExecutable();
1206 WriteValidationScript();
1207 if (!CreateJDL()) return;
1208 if (TestBit(AliAnalysisGrid::kOffline)) return;
1209 if (TestBit(AliAnalysisGrid::kTest)) {
1210 // Locally testing the analysis
1211 Info("StartAnalysis", "\n_______________________________________________________________________ \
1212 \n Running analysis script in a daughter shell as on a worker node \
1213 \n_______________________________________________________________________");
1214 TObjArray *list = fOutputFiles.Tokenize(" ");
1215 TIter next(list);
1216 TObjString *str;
1217 TString output_file;
1218 while((str=(TObjString*)next())) {
1219 output_file = str->GetString();
1220 Int_t index = output_file.Index("@");
1221 if (index > 0) output_file.Remove(index);
43da816a 1222 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1223 }
1224 delete list;
1225 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1226 gSystem->Exec("bash validate.sh");
1227// gSystem->Exec("cat stdout");
1228 return;
1229 }
a8739e8a 1230 // Submit AliEn job(s)
c57f56b7 1231 CdWork();
a8739e8a 1232 TGridResult *res;
c57f56b7 1233 TString jobID = "";
dd74a515 1234 if (!fRunRange[0]) {
1235 // Submit a given xml or a set of runs
a8739e8a 1236 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1237 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1238 if (res) {
1239 const char *cjobId = res->GetKey(0,"jobId");
1240 if (!cjobId) {
1241 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1242 return;
1243 } else {
1244 Info("StartAnalysis", "\n_______________________________________________________________________ \
1245 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1246 \n_______________________________________________________________________",
1247 fJDLName.Data(), cjobId);
1248 jobID = cjobId;
1249 }
1250 delete res;
1251 }
1252 } else {
dd74a515 1253 // Submit for a range of runs.
a8739e8a 1254 TString sjdl;
1255 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
b15e08c1 1256 if (!fInputFiles->At(irun-fRunRange[0])) break;
a8739e8a 1257 sjdl = fJDLName;
1258 sjdl.ReplaceAll(".jdl", Form("_%03d.jdl", irun-fRunRange[0]));
1259 res = gGrid->Command(Form("submit %s", sjdl.Data()));
1260 printf("*************************** %s\n",Form("submit %s", sjdl.Data()));
1261 if (res) {
1262 const char *cjobId1 = res->GetKey(0,"jobId");
1263 if (!cjobId1) {
1264 Error("StartAnalysis", "Your JDL %s could not be submitted", sjdl.Data());
1265 return;
1266 } else {
1267 Info("StartAnalysis", "\n_______________________________________________________________________ \
1268 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1269 \n_______________________________________________________________________",
1270 sjdl.Data(), cjobId1);
1271 jobID += cjobId1;
1272 jobID += " ";
1273 }
1274 delete res;
1275 }
1276 }
c57f56b7 1277 }
a8739e8a 1278
c57f56b7 1279 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1280 \n You may exit at any time and terminate the job later using the option <terminate> \
1281 \n ##################################################################################", jobID.Data());
bb885a9e 1282 gSystem->Exec("aliensh");
c57f56b7 1283}
1284
1285//______________________________________________________________________________
1286void AliAnalysisAlien::WriteAnalysisFile()
1287{
1288// Write current analysis manager into the file analysis.root
1289 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1290 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1291 if (!mgr || !mgr->IsInitialized()) {
1292 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1293 return;
1294 }
1295 // Check analysis type
1296 TObject *handler;
1297 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1298 handler = (TObject*)mgr->GetInputEventHandler();
1299 if (handler) {
1300 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1301 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1302 }
1303 TDirectory *cdir = gDirectory;
1304 TFile *file = TFile::Open("analysis.root", "RECREATE");
1305 if (file) {
1306 mgr->Write();
1307 delete file;
1308 }
1309 if (cdir) cdir->cd();
1310 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1311 }
1312 Bool_t copy = kTRUE;
1313 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1314 if (copy) {
1315 CdWork();
1316 TString workdir = gGrid->GetHomeDirectory();
1317 workdir += fGridWorkingDir;
1318 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1319 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1320 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1321 }
1322}
1323
1324//______________________________________________________________________________
1325void AliAnalysisAlien::WriteAnalysisMacro()
1326{
1327// Write the analysis macro that will steer the analysis in grid mode.
1328 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1329 ofstream out;
1330 out.open(fAnalysisMacro.Data(), ios::out);
1331 if (!out.good()) {
1332 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1333 return;
1334 }
1335 TString func = fAnalysisMacro;
1336 TString type = "ESD";
1337 TString comment = "// Analysis using ";
1338 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1339 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1340 type = "AOD";
1341 comment += "AOD";
1342 }
0df6ccf2 1343 if (type!="AOD" && fFriendChainName!="") {
1344 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1345 return;
1346 }
c57f56b7 1347 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1348 else comment += " data";
1349 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1350 func.ReplaceAll(".C", "");
1351 out << "void " << func.Data() << "()" << endl;
1352 out << "{" << endl;
1353 out << comment.Data() << endl;
1354 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1355 out << " TStopwatch timer;" << endl;
1356 out << " timer.Start();" << endl << endl;
c57f56b7 1357 out << "// load base root libraries" << endl;
1358 out << " gSystem->Load(\"libTree\");" << endl;
1359 out << " gSystem->Load(\"libGeom\");" << endl;
1360 out << " gSystem->Load(\"libVMC\");" << endl;
1361 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1362 out << "// Load analysis framework libraries" << endl;
4e5c5506 1363 if (!fPackages) {
4e5c5506 1364 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1365 out << " gSystem->Load(\"libESD\");" << endl;
1366 out << " gSystem->Load(\"libAOD\");" << endl;
1367 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1368 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1369 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1370 } else {
4e5c5506 1371 TIter next(fPackages);
1372 TObject *obj;
57377eb5 1373 TString pkgname;
1374 Bool_t hasSTEERBase = kFALSE;
1375 Bool_t hasESD = kFALSE;
1376 Bool_t hasAOD = kFALSE;
1377 Bool_t hasANALYSIS = kFALSE;
1378 Bool_t hasANALYSISalice = kFALSE;
1379 Bool_t hasCORRFW = kFALSE;
1380 while ((obj=next())) {
1381 pkgname = obj->GetName();
4478e6f1 1382 if (pkgname == "STEERBase" ||
1383 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1384 if (pkgname == "ESD" ||
1385 pkgname == "ESD.par") hasESD = kTRUE;
1386 if (pkgname == "AOD" ||
1387 pkgname == "AOD.par") hasAOD = kTRUE;
1388 if (pkgname == "ANALYSIS" ||
1389 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1390 if (pkgname == "ANALYSISalice" ||
1391 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1392 if (pkgname == "CORRFW" ||
1393 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1394 }
1395 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1396 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1397 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1398 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1399 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1400 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1401 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1402 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1403 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1404 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1405 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1406 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1407 out << "// Compile other par packages" << endl;
1408 next.Reset();
fcc9bb6f 1409 while ((obj=next())) {
1410 pkgname = obj->GetName();
4478e6f1 1411 if (pkgname == "STEERBase" ||
1412 pkgname == "STEERBase.par" ||
1413 pkgname == "ESD" ||
1414 pkgname == "ESD.par" ||
1415 pkgname == "AOD" ||
1416 pkgname == "AOD.par" ||
1417 pkgname == "ANALYSIS" ||
1418 pkgname == "ANALYSIS.par" ||
1419 pkgname == "ANALYSISalice" ||
1420 pkgname == "ANALYSISalice.par" ||
1421 pkgname == "CORRFW" ||
1422 pkgname == "CORRFW.par") continue;
4e5c5506 1423 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1424 }
4e5c5506 1425 }
43da816a 1426 out << "// include path" << endl;
1427 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1428 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1429 if (fAdditionalLibs.Length()) {
1430 out << "// Add aditional AliRoot libraries" << endl;
1431 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1432 TIter next(list);
1433 TObjString *str;
1434 while((str=(TObjString*)next())) {
1435 if (str->GetString().Contains(".so"))
1436 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1437 }
1438 if (list) delete list;
1439 }
1440 out << endl;
c57f56b7 1441 out << "// analysis source to be compiled at runtime (if any)" << endl;
1442 if (fAnalysisSource.Length()) {
1443 TObjArray *list = fAnalysisSource.Tokenize(" ");
1444 TIter next(list);
1445 TObjString *str;
1446 while((str=(TObjString*)next())) {
1447 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1448 }
1449 if (list) delete list;
1450 }
1451 out << endl;
1452 out << "// connect to AliEn and make the chain" << endl;
1453 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1454 if (IsUsingTags()) {
1455 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1456 } else {
1457 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1458 }
1459 out << "// read the analysis manager from file" << endl;
1460 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1461 out << " if (!file) return;" << endl;
1462 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1463 out << " AliAnalysisManager *mgr = 0;" << endl;
1464 out << " TKey *key;" << endl;
1465 out << " while ((key=(TKey*)nextkey())) {" << endl;
1466 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1467 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1468 out << " };" << endl;
1469 out << " if (!mgr) {" << endl;
1470 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1471 out << " return;" << endl;
1472 out << " }" << endl << endl;
1473 out << " mgr->PrintStatus();" << endl;
1474 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1475 out << " timer.Stop();" << endl;
1476 out << " timer.Print();" << endl;
c57f56b7 1477 out << "}" << endl << endl;
1478 if (IsUsingTags()) {
1479 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1480 out << "{" << endl;
1481 out << "// Create a chain using tags from the xml file." << endl;
1482 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1483 out << " if (!coll) {" << endl;
1484 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1485 out << " return NULL;" << endl;
1486 out << " }" << endl;
1487 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1488 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1489 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1490 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1491 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1492 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1493 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1494 out << " // Check if the cuts configuration file was provided" << endl;
1495 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1496 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1497 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1498 out << " }" << endl;
0df6ccf2 1499 if (fFriendChainName=="") {
1500 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1501 } else {
1502 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1503 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1504 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1505 }
c57f56b7 1506 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1507 out << " chain->ls();" << endl;
1508 out << " return chain;" << endl;
fcc9bb6f 1509 out << "}" << endl << endl;
c57f56b7 1510 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1511 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1512 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1513 msg += " AliLHCTagCuts *lhcCuts,\n";
1514 msg += " AliDetectorTagCuts *detCuts,\n";
1515 msg += " AliEventTagCuts *evCuts)";
1516 Info("WriteAnalysisMacro", msg.Data());
1517 }
0df6ccf2 1518 }
1519 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1520 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1521 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1522 out << "{" << endl;
1523 out << "// Create a chain using url's from xml file" << endl;
1524 out << " TString treename = type;" << endl;
1525 out << " treename.ToLower();" << endl;
1526 out << " treename += \"Tree\";" << endl;
e02fee64 1527 out << " printf(\"***************************************\\n\");" << endl;
1528 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1529 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1530 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1531 out << " if (!coll) {" << endl;
1532 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1533 out << " return NULL;" << endl;
1534 out << " }" << endl;
1535 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1536 if(fFriendChainName!="") {
1537 out << " TChain *chainFriend = new TChain(treename);" << endl;
1538 }
c57f56b7 1539 out << " coll->Reset();" << endl;
0df6ccf2 1540 out << " while (coll->Next()) {" << endl;
1541 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1542 if(fFriendChainName!="") {
1543 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1544 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1545 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1546 out << " chainFriend->Add(fileFriend.Data());" << endl;
1547 }
1548 out << " }" << endl;
c57f56b7 1549 out << " if (!chain->GetNtrees()) {" << endl;
1550 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1551 out << " return NULL;" << endl;
1552 out << " }" << endl;
0df6ccf2 1553 if(fFriendChainName!="") {
1554 out << " chain->AddFriend(chainFriend);" << endl;
1555 }
c57f56b7 1556 out << " return chain;" << endl;
fcc9bb6f 1557 out << "}" << endl << endl;
c57f56b7 1558 }
4e5c5506 1559 if (fPackages) {
fcc9bb6f 1560 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1561 out << "Bool_t SetupPar(const char *package) {" << endl;
1562 out << "// Compile the package and set it up." << endl;
1563 out << " TString pkgdir = package;" << endl;
1564 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1565 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1566 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1567 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1568 out << " // Check for BUILD.sh and execute" << endl;
1569 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1570 out << " printf(\"*******************************\\n\");" << endl;
1571 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1572 out << " printf(\"*******************************\\n\");" << endl;
1573 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1574 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1575 out << " gSystem->ChangeDirectory(cdir);" << endl;
1576 out << " return kFALSE;" << endl;
1577 out << " }" << endl;
1578 out << " } else {" << endl;
fcc9bb6f 1579 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1580 out << " gSystem->ChangeDirectory(cdir);" << endl;
1581 out << " return kFALSE;" << endl;
1582 out << " }" << endl;
1583 out << " // Check for SETUP.C and execute" << endl;
1584 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1585 out << " printf(\"*******************************\\n\");" << endl;
1586 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1587 out << " printf(\"*******************************\\n\");" << endl;
1588 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1589 out << " } else {" << endl;
fcc9bb6f 1590 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1591 out << " gSystem->ChangeDirectory(cdir);" << endl;
1592 out << " return kFALSE;" << endl;
1593 out << " }" << endl;
1594 out << " // Restore original workdir" << endl;
1595 out << " gSystem->ChangeDirectory(cdir);" << endl;
1596 out << " return kTRUE;" << endl;
1597 out << "}" << endl;
1598 }
c57f56b7 1599 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1600 }
1601 Bool_t copy = kTRUE;
1602 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1603 if (copy) {
1604 CdWork();
1605 TString workdir = gGrid->GetHomeDirectory();
1606 workdir += fGridWorkingDir;
1607 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1608 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1609 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1610 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1611 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1612 }
1613 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1614 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1615 }
1616}
1617
1618//______________________________________________________________________________
1619void AliAnalysisAlien::WriteExecutable()
1620{
1621// Generate the alien executable script.
1622 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1623 ofstream out;
1624 out.open(fExecutable.Data(), ios::out);
1625 if (out.bad()) {
1626 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1627 return;
1628 }
1629 out << "#!/bin/bash" << endl;
1630 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1631 out << "echo \"=========================================\"" << endl;
1632 out << "echo \"############## PATH : ##############\"" << endl;
1633 out << "echo $PATH" << endl;
1634 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1635 out << "echo $LD_LIBRARY_PATH" << endl;
1636 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1637 out << "echo $ROOTSYS" << endl;
1638 out << "echo \"############## which root : ##############\"" << endl;
1639 out << "which root" << endl;
1640 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1641 out << "echo $ALICE_ROOT" << endl;
1642 out << "echo \"############## which aliroot : ##############\"" << endl;
1643 out << "which aliroot" << endl;
1644 out << "echo \"=========================================\"" << endl << endl;
1645// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1646 out << "root -b -q ";
1647 out << fAnalysisMacro.Data() << endl << endl;
1648 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1649 }
1650 Bool_t copy = kTRUE;
1651 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1652 if (copy) {
1653 CdWork();
1654 TString workdir = gGrid->GetHomeDirectory();
1655 workdir += fGridWorkingDir;
1656 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1657 if (FileExists(executable)) gGrid->Rm(executable);
1658 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1659 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1660 }
1661}
1662
1663//______________________________________________________________________________
1664void AliAnalysisAlien::WriteValidationScript()
1665{
1666// Generate the alien validation script.
1667 // Generate the validation script
1668 TObjString *os;
1669 if (!Connect()) {
1670 Error("WriteValidationScript", "Alien connection required");
1671 return;
1672 }
1673 TString out_stream = "";
1674 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1675 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1676 ofstream out;
1677 out.open("validate.sh", ios::out);
1678 out << "#!/bin/bash" << endl;
1679 out << "##################################################" << endl;
1680 out << "validateout=`dirname $0`" << endl;
1681 out << "validatetime=`date`" << endl;
1682 out << "validated=\"0\";" << endl;
1683 out << "error=0" << endl;
1684 out << "if [ -z $validateout ]" << endl;
1685 out << "then" << endl;
1686 out << " validateout=\".\"" << endl;
1687 out << "fi" << endl << endl;
1688 out << "cd $validateout;" << endl;
1689 out << "validateworkdir=`pwd`;" << endl << endl;
1690 out << "echo \"*******************************************************\"" << out_stream << endl;
1691 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1692 out << "" << endl;
1693 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1694 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1695 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1696 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1697 out << "ls -la ./" << out_stream << endl;
1698 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1699 out << "##################################################" << endl;
ebec370a 1700
1701 out << "" << endl;
1702 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
1703 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
1704 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
1705 out << "" << endl;
1706
1707 out << "if [ ! -f stderr ] ; then" << endl;
1708 out << " error=1" << endl;
1709 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
1710 out << " echo \"Error = $error\" " << out_stream << endl;
1711 out << "fi" << endl;
1712
1713 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
1714 out << " error=1" << endl;
1715 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
1716 out << " echo \"$parArch\" " << out_stream << endl;
1717 out << " echo \"Error = $error\" " << out_stream << endl;
1718 out << "fi" << endl;
1719
1720 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
1721 out << " error=1" << endl;
1722 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
1723 out << " echo \"$segViol\" " << out_stream << endl;
1724 out << " echo \"Error = $error\" " << out_stream << endl;
1725 out << "fi" << endl;
1726
1727 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
1728 out << " error=1" << endl;
1729 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
1730 out << " echo \"$segFault\" " << out_stream << endl;
1731 out << " echo \"Error = $error\" " << out_stream << endl;
1732 out << "fi" << endl;
1733
1734 // Part dedicated to the specific analyses running into the train
1735
c57f56b7 1736 TObjArray *arr = fOutputFiles.Tokenize(" ");
1737 TIter next1(arr);
1738 TString output_file;
1739 while ((os=(TObjString*)next1())) {
1740 output_file = os->GetString();
1741 Int_t index = output_file.Index("@");
1742 if (index > 0) output_file.Remove(index);
1743 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1744 out << " error=1" << endl;
1745 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1746 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1747 out << "fi" << endl;
1748 }
1749 delete arr;
1750 out << "if [ $error = 0 ] ; then" << endl;
1751 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1752 out << "fi" << endl;
1753
1754 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1755 out << "echo \"*******************************************************\"" << out_stream << endl;
1756 out << "cd -" << endl;
1757 out << "exit $error" << endl;
1758 }
1759 Bool_t copy = kTRUE;
1760 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1761 if (copy) {
1762 CdWork();
1763 TString workdir = gGrid->GetHomeDirectory();
1764 workdir += fGridWorkingDir;
1765 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1766 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1767 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
1768 }
1769}