]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
New version including PWG2 AddTaskCentral, modified KINK resonances like sign macros...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
0a1c1f7f 59 fExecutableCommand(),
c57f56b7 60 fArguments(),
61 fAnalysisMacro(),
62 fAnalysisSource(),
63 fAdditionalLibs(),
64 fSplitMode(),
65 fAPIVersion(),
66 fROOTVersion(),
67 fAliROOTVersion(),
648174cf 68 fExternalPackages(),
c57f56b7 69 fUser(),
70 fGridWorkingDir(),
71 fGridDataDir(),
72 fDataPattern(),
73 fGridOutputDir(),
74 fOutputArchive(),
75 fOutputFiles(),
76 fInputFormat(),
e7c71df0 77 fDatasetName(),
c57f56b7 78 fJDLName(),
bb885a9e 79 fMergeExcludes(),
f965131e 80 fIncludePath(),
bb885a9e 81 fCloseSE(),
0df6ccf2 82 fFriendChainName(),
c6cb3634 83 fJobTag(),
648174cf 84 fOutputSingle(),
4e5c5506 85 fInputFiles(0),
86 fPackages(0)
c57f56b7 87{
88// Dummy ctor.
89 SetDefaults();
90}
91
92//______________________________________________________________________________
93AliAnalysisAlien::AliAnalysisAlien(const char *name)
94 :AliAnalysisGrid(name),
95 fGridJDL(NULL),
96 fPrice(0),
97 fTTL(0),
98 fSplitMaxInputFileNumber(0),
99 fMaxInitFailed(0),
100 fMasterResubmitThreshold(0),
bb885a9e 101 fNtestFiles(0),
319593fb 102 fNrunsPerMaster(0),
16a4353c 103 fMaxMergeFiles(0),
d2a409b2 104 fNsubmitted(0),
a3e84053 105 fProductionMode(0),
c57f56b7 106 fRunNumbers(),
107 fExecutable(),
0a1c1f7f 108 fExecutableCommand(),
c57f56b7 109 fArguments(),
110 fAnalysisMacro(),
111 fAnalysisSource(),
112 fAdditionalLibs(),
113 fSplitMode(),
114 fAPIVersion(),
115 fROOTVersion(),
116 fAliROOTVersion(),
648174cf 117 fExternalPackages(),
c57f56b7 118 fUser(),
119 fGridWorkingDir(),
120 fGridDataDir(),
121 fDataPattern(),
122 fGridOutputDir(),
123 fOutputArchive(),
124 fOutputFiles(),
125 fInputFormat(),
e7c71df0 126 fDatasetName(),
c57f56b7 127 fJDLName(),
bb885a9e 128 fMergeExcludes(),
f965131e 129 fIncludePath(),
bb885a9e 130 fCloseSE(),
0df6ccf2 131 fFriendChainName(),
c6cb3634 132 fJobTag(),
648174cf 133 fOutputSingle(),
4e5c5506 134 fInputFiles(0),
135 fPackages(0)
c57f56b7 136{
137// Default ctor.
138 SetDefaults();
139}
140
141//______________________________________________________________________________
142AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
143 :AliAnalysisGrid(other),
144 fGridJDL(NULL),
145 fPrice(other.fPrice),
146 fTTL(other.fTTL),
147 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
148 fMaxInitFailed(other.fMaxInitFailed),
149 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 150 fNtestFiles(other.fNtestFiles),
319593fb 151 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 152 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 153 fNsubmitted(other.fNsubmitted),
a3e84053 154 fProductionMode(other.fProductionMode),
c57f56b7 155 fRunNumbers(other.fRunNumbers),
156 fExecutable(other.fExecutable),
0a1c1f7f 157 fExecutableCommand(other.fExecutableCommand),
c57f56b7 158 fArguments(other.fArguments),
159 fAnalysisMacro(other.fAnalysisMacro),
160 fAnalysisSource(other.fAnalysisSource),
161 fAdditionalLibs(other.fAdditionalLibs),
162 fSplitMode(other.fSplitMode),
163 fAPIVersion(other.fAPIVersion),
164 fROOTVersion(other.fROOTVersion),
165 fAliROOTVersion(other.fAliROOTVersion),
648174cf 166 fExternalPackages(other.fExternalPackages),
c57f56b7 167 fUser(other.fUser),
168 fGridWorkingDir(other.fGridWorkingDir),
169 fGridDataDir(other.fGridDataDir),
170 fDataPattern(other.fDataPattern),
171 fGridOutputDir(other.fGridOutputDir),
172 fOutputArchive(other.fOutputArchive),
173 fOutputFiles(other.fOutputFiles),
174 fInputFormat(other.fInputFormat),
e7c71df0 175 fDatasetName(other.fDatasetName),
c57f56b7 176 fJDLName(other.fJDLName),
bb885a9e 177 fMergeExcludes(other.fMergeExcludes),
f965131e 178 fIncludePath(other.fIncludePath),
bb885a9e 179 fCloseSE(other.fCloseSE),
0df6ccf2 180 fFriendChainName(other.fFriendChainName),
c6cb3634 181 fJobTag(other.fJobTag),
648174cf 182 fOutputSingle(other.fOutputSingle),
4e5c5506 183 fInputFiles(0),
184 fPackages(0)
c57f56b7 185{
186// Copy ctor.
187 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 188 fRunRange[0] = other.fRunRange[0];
189 fRunRange[1] = other.fRunRange[1];
c57f56b7 190 if (other.fInputFiles) {
191 fInputFiles = new TObjArray();
192 TIter next(other.fInputFiles);
193 TObject *obj;
194 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
195 fInputFiles->SetOwner();
196 }
4e5c5506 197 if (other.fPackages) {
198 fPackages = new TObjArray();
199 TIter next(other.fPackages);
200 TObject *obj;
201 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
202 fPackages->SetOwner();
203 }
c57f56b7 204}
205
206//______________________________________________________________________________
207AliAnalysisAlien::~AliAnalysisAlien()
208{
209// Destructor.
210 if (fGridJDL) delete fGridJDL;
211 if (fInputFiles) delete fInputFiles;
4e5c5506 212 if (fPackages) delete fPackages;
c57f56b7 213}
214
215//______________________________________________________________________________
216AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
217{
218// Assignment.
219 if (this != &other) {
220 AliAnalysisGrid::operator=(other);
221 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
222 fPrice = other.fPrice;
223 fTTL = other.fTTL;
224 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
225 fMaxInitFailed = other.fMaxInitFailed;
226 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 227 fNtestFiles = other.fNtestFiles;
a3e84053 228 fNrunsPerMaster = other.fNrunsPerMaster;
229 fMaxMergeFiles = other.fMaxMergeFiles;
230 fNsubmitted = other.fNsubmitted;
231 fProductionMode = other.fProductionMode;
c57f56b7 232 fRunNumbers = other.fRunNumbers;
233 fExecutable = other.fExecutable;
0a1c1f7f 234 fExecutableCommand = other.fExecutableCommand;
c57f56b7 235 fArguments = other.fArguments;
236 fAnalysisMacro = other.fAnalysisMacro;
237 fAnalysisSource = other.fAnalysisSource;
238 fAdditionalLibs = other.fAdditionalLibs;
239 fSplitMode = other.fSplitMode;
240 fAPIVersion = other.fAPIVersion;
241 fROOTVersion = other.fROOTVersion;
242 fAliROOTVersion = other.fAliROOTVersion;
648174cf 243 fExternalPackages = other.fExternalPackages;
c57f56b7 244 fUser = other.fUser;
245 fGridWorkingDir = other.fGridWorkingDir;
246 fGridDataDir = other.fGridDataDir;
247 fDataPattern = other.fDataPattern;
248 fGridOutputDir = other.fGridOutputDir;
249 fOutputArchive = other.fOutputArchive;
250 fOutputFiles = other.fOutputFiles;
251 fInputFormat = other.fInputFormat;
e7c71df0 252 fDatasetName = other.fDatasetName;
c57f56b7 253 fJDLName = other.fJDLName;
bb885a9e 254 fMergeExcludes = other.fMergeExcludes;
f965131e 255 fIncludePath = other.fIncludePath;
bb885a9e 256 fCloseSE = other.fCloseSE;
0df6ccf2 257 fFriendChainName = other.fFriendChainName;
c6cb3634 258 fJobTag = other.fJobTag;
648174cf 259 fOutputSingle = other.fOutputSingle;
c57f56b7 260 if (other.fInputFiles) {
261 fInputFiles = new TObjArray();
262 TIter next(other.fInputFiles);
263 TObject *obj;
264 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
265 fInputFiles->SetOwner();
266 }
4e5c5506 267 if (other.fPackages) {
268 fPackages = new TObjArray();
269 TIter next(other.fPackages);
270 TObject *obj;
271 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
272 fPackages->SetOwner();
273 }
c57f56b7 274 }
275 return *this;
276}
277
f965131e 278//______________________________________________________________________________
279void AliAnalysisAlien::AddIncludePath(const char *path)
280{
281// Add include path in the remote analysis macro.
282 TString p(path);
283 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
284 else fIncludePath += Form("-I%s ", path);
285}
286
c57f56b7 287//______________________________________________________________________________
288void AliAnalysisAlien::AddRunNumber(Int_t run)
289{
290// Add a run number to the list of runs to be processed.
291 if (fRunNumbers.Length()) fRunNumbers += " ";
292 fRunNumbers += Form("%d", run);
293}
294
ee75cfc3 295//______________________________________________________________________________
296void AliAnalysisAlien::AddRunNumber(const char* run)
297{
298// Add a run number to the list of runs to be processed.
299 if (fRunNumbers.Length()) fRunNumbers += " ";
300 fRunNumbers += run;
301}
302
c57f56b7 303//______________________________________________________________________________
304void AliAnalysisAlien::AddDataFile(const char *lfn)
305{
306// Adds a data file to the input to be analysed. The file should be a valid LFN
307// or point to an existing file in the alien workdir.
308 if (!fInputFiles) fInputFiles = new TObjArray();
309 fInputFiles->Add(new TObjString(lfn));
310}
648174cf 311
312//______________________________________________________________________________
313void AliAnalysisAlien::AddExternalPackage(const char *package)
314{
315// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
316 if (fExternalPackages) fExternalPackages += " ";
317 fExternalPackages += package;
318}
319
c57f56b7 320//______________________________________________________________________________
321Bool_t AliAnalysisAlien::Connect()
322{
323// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
324 if (gGrid && gGrid->IsConnected()) return kTRUE;
325 if (!gSystem->Getenv("alien_API_USER")) {
326 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
327 gSystem->Getenv("UID"));
328 return kFALSE;
329 }
330 if (!gGrid) {
331 Info("Connect", "Trying to connect to AliEn ...");
332 TGrid::Connect("alien://");
333 }
334 if (!gGrid || !gGrid->IsConnected()) {
335 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
336 return kFALSE;
337 }
338 fUser = gGrid->GetUser();
339 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
340 return kTRUE;
341}
342
343//______________________________________________________________________________
344void AliAnalysisAlien::CdWork()
345{
346// Check validity of alien workspace. Create directory if possible.
347 if (!Connect()) {
348 Error("CdWork", "Alien connection required");
349 return;
350 }
351 TString homedir = gGrid->GetHomeDirectory();
352 TString workdir = homedir + fGridWorkingDir;
923e2ca5 353 if (DirectoryExists(workdir)) {
354 gGrid->Cd(workdir);
355 return;
356 }
357 // Work directory not existing - create it
358 gGrid->Cd(homedir);
359 if (gGrid->Mkdir(workdir)) {
360 gGrid->Cd(fGridWorkingDir);
361 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
362 } else {
363 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
364 workdir.Data(), homedir.Data());
365 fGridWorkingDir = "";
366 }
c57f56b7 367}
368
369//______________________________________________________________________________
370Bool_t AliAnalysisAlien::CheckInputData()
371{
372// Check validity of input data. If necessary, create xml files.
a8739e8a 373 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 374 if (!fGridDataDir.Length()) {
375 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
376 return kFALSE;
377 }
378 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
379 return kTRUE;
c57f56b7 380 }
381 // Process declared files
382 Bool_t is_collection = kFALSE;
383 Bool_t is_xml = kFALSE;
384 Bool_t use_tags = kFALSE;
385 Bool_t checked = kFALSE;
386 CdWork();
387 TString file;
388 TString workdir = gGrid->GetHomeDirectory();
389 workdir += fGridWorkingDir;
390 if (fInputFiles) {
391 TObjString *objstr;
392 TIter next(fInputFiles);
393 while ((objstr=(TObjString*)next())) {
394 file = workdir;
395 file += "/";
396 file += objstr->GetString();
397 // Store full lfn path
398 if (FileExists(file)) objstr->SetString(file);
399 else {
400 file = objstr->GetName();
401 if (!FileExists(objstr->GetName())) {
402 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
403 objstr->GetName(), workdir.Data());
404 return kFALSE;
405 }
406 }
407 Bool_t iscoll, isxml, usetags;
408 CheckDataType(file, iscoll, isxml, usetags);
409 if (!checked) {
410 checked = kTRUE;
411 is_collection = iscoll;
412 is_xml = isxml;
413 use_tags = usetags;
414 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
415 } else {
416 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
417 Error("CheckInputData", "Some conflict was found in the types of inputs");
418 return kFALSE;
419 }
420 }
421 }
422 }
423 // Process requested run numbers
a8739e8a 424 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 425 // Check validity of alien data directory
426 if (!fGridDataDir.Length()) {
427 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
428 return kFALSE;
429 }
923e2ca5 430 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 431 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
432 return kFALSE;
433 }
434 if (is_collection) {
435 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
436 return kFALSE;
437 }
438
439 if (checked && !is_xml) {
440 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
441 return kFALSE;
442 }
443 // Check validity of run number(s)
444 TObjArray *arr;
445 TObjString *os;
319593fb 446 Int_t nruns = 0;
447 TString schunk;
c57f56b7 448 TString path;
449 if (!checked) {
450 checked = kTRUE;
451 use_tags = fDataPattern.Contains("tag");
452 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
453 }
454 if (use_tags != fDataPattern.Contains("tag")) {
455 Error("CheckInputData", "Cannot mix input files using/not using tags");
456 return kFALSE;
457 }
458 if (fRunNumbers.Length()) {
a8739e8a 459 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 460 arr = fRunNumbers.Tokenize(" ");
461 TIter next(arr);
462 while ((os=(TObjString*)next())) {
463 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 464 if (!DirectoryExists(path)) {
465 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 466 continue;
c57f56b7 467 }
468 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
469 TString msg = "\n##### file: ";
470 msg += path;
471 msg += " type: xml_collection;";
472 if (use_tags) msg += " using_tags: Yes";
473 else msg += " using_tags: No";
474 Info("CheckDataType", msg.Data());
319593fb 475 if (fNrunsPerMaster<2) {
d2a409b2 476 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 477 } else {
478 nruns++;
479 if (((nruns-1)%fNrunsPerMaster) == 0) {
480 schunk = os->GetString();
481 }
482 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
483 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 484 AddDataFile(schunk);
319593fb 485 }
c57f56b7 486 }
487 delete arr;
a8739e8a 488 } else {
489 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
490 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
491 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 492 if (!DirectoryExists(path)) {
493// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 494 continue;
495 }
496 path = Form("%s/%d.xml", workdir.Data(),irun);
497 TString msg = "\n##### file: ";
498 msg += path;
499 msg += " type: xml_collection;";
500 if (use_tags) msg += " using_tags: Yes";
501 else msg += " using_tags: No";
502 Info("CheckDataType", msg.Data());
319593fb 503 if (fNrunsPerMaster<2) {
d2a409b2 504 AddDataFile(Form("%d.xml",irun));
319593fb 505 } else {
506 nruns++;
507 if (((nruns-1)%fNrunsPerMaster) == 0) {
508 schunk = Form("%d", irun);
509 }
510 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
511 schunk += Form("_%d.xml", irun);
d2a409b2 512 AddDataFile(schunk);
319593fb 513 }
a8739e8a 514 }
c57f56b7 515 }
516 return kTRUE;
517}
518
519//______________________________________________________________________________
520Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
521{
522// Create dataset for the grid data directory + run number.
523 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
524 if (!Connect()) {
525 Error("CreateDataset", "Cannot create dataset with no grid connection");
526 return kFALSE;
527 }
528
529 // Cd workspace
530 CdWork();
531 TString workdir = gGrid->GetHomeDirectory();
532 workdir += fGridWorkingDir;
533
534 // Compose the 'find' command arguments
535 TString command;
536 TString options = "-x collection ";
bb885a9e 537 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 538 TString conditions = "";
539
540 TString file;
541 TString path;
319593fb 542 Int_t nruns = 0;
543 TString schunk;
ab254fd1 544 TGridCollection *cbase=0, *cadd=0;
d2a409b2 545 if (!fRunNumbers.Length() && !fRunRange[0]) {
546 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
547 // Make a single data collection from data directory.
548 path = fGridDataDir;
923e2ca5 549 if (!DirectoryExists(path)) {
d2a409b2 550 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
551 return kFALSE;
552 }
923e2ca5 553// CdWork();
d2a409b2 554 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
555 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 556 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 557 command = "find ";
558 command += options;
559 command += path;
560 command += " ";
561 command += pattern;
562 command += conditions;
84fcd93f 563 printf("command: %s\n", command.Data());
d2a409b2 564 TGridResult *res = gGrid->Command(command);
565 if (res) delete res;
566 // Write standard output to file
567 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
568 }
569 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
570 // Copy xml file to alien space
571 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
572 if (!FileExists(file)) {
573 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
574 return kFALSE;
575 }
576 // Update list of files to be processed.
577 }
578 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
579 return kTRUE;
580 }
c57f56b7 581 // Several runs
a8739e8a 582 if (fRunNumbers.Length()) {
583 TObjArray *arr = fRunNumbers.Tokenize(" ");
584 TObjString *os;
585 TIter next(arr);
586 while ((os=(TObjString*)next())) {
587 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 588 if (!DirectoryExists(path)) continue;
589// CdWork();
a8739e8a 590 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
591 else file = Form("%s.xml", os->GetString().Data());
319593fb 592 // If local collection file does not exist, create it via 'find' command.
593 if (gSystem->AccessPathName(file)) {
594 command = "find ";
595 command += options;
596 command += path;
597 command += pattern;
598 command += conditions;
599 TGridResult *res = gGrid->Command(command);
600 if (res) delete res;
601 // Write standard output to file
602 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
603 }
a8739e8a 604 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 605 // Check if there is one run per master job.
606 if (fNrunsPerMaster<2) {
607 if (FileExists(file)) {
608 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
609 continue;
610 }
611 // Copy xml file to alien space
612 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
613 if (!FileExists(file)) {
614 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
615 delete arr;
616 return kFALSE;
617 }
618 } else {
619 nruns++;
620 if (((nruns-1)%fNrunsPerMaster) == 0) {
621 schunk = os->GetString();
622 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
623 } else {
624 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 625 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 626 cbase->Add(cadd);
627 delete cadd;
628 }
629 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
630 continue;
631 }
632 schunk += Form("_%s.xml", os->GetString().Data());
633 if (FileExists(schunk)) {
634 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
635 continue;
636 }
84fcd93f 637 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 638 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 639 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 640 if (!FileExists(schunk)) {
641 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
642 delete arr;
643 return kFALSE;
644 }
645 }
a8739e8a 646 }
647 delete arr;
648 } else {
649 // Process a full run range.
650 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
651 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 652 if (!DirectoryExists(path)) continue;
653// CdWork();
a8739e8a 654 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
655 else file = Form("%d.xml", irun);
319593fb 656 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 657 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
658// gGrid->Rm(file);
659 continue;
660 }
319593fb 661 // If local collection file does not exist, create it via 'find' command.
662 if (gSystem->AccessPathName(file)) {
663 command = "find ";
664 command += options;
665 command += path;
666 command += pattern;
667 command += conditions;
668 TGridResult *res = gGrid->Command(command);
669 if (res) delete res;
670 // Write standard output to file
671 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
672 }
a8739e8a 673 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 674 // Check if there is one run per master job.
675 if (fNrunsPerMaster<2) {
676 if (FileExists(file)) {
677 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
678 continue;
679 }
680 // Copy xml file to alien space
681 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
682 if (!FileExists(file)) {
683 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
684 return kFALSE;
685 }
686 } else {
687 nruns++;
95e5b448 688 // Check if the collection for the chunk exist locally.
689 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
690 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 691 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 692 if (((nruns-1)%fNrunsPerMaster) == 0) {
693 schunk = Form("%d", irun);
694 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
695 } else {
696 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
697 cbase->Add(cadd);
698 delete cadd;
699 }
700 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
701 continue;
702 }
703 schunk += Form("_%d.xml", irun);
704 if (FileExists(schunk)) {
705 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
706 continue;
707 }
84fcd93f 708 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 709 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 710 if (FileExists(schunk)) {
711 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
712 continue;
713 }
319593fb 714 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
715 if (!FileExists(schunk)) {
716 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
717 return kFALSE;
718 }
719 }
c57f56b7 720 }
a8739e8a 721 }
c57f56b7 722 return kTRUE;
723}
724
725//______________________________________________________________________________
726Bool_t AliAnalysisAlien::CreateJDL()
727{
728// Generate a JDL file according to current settings. The name of the file is
729// specified by fJDLName.
730 Bool_t error = kFALSE;
731 TObjArray *arr = 0;
732 Bool_t copy = kTRUE;
733 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
734 Bool_t generate = kTRUE;
735 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
736 if (!Connect()) {
737 Error("CreateJDL", "Alien connection required");
738 return kFALSE;
739 }
740 // Check validity of alien workspace
741 CdWork();
742 TString workdir = gGrid->GetHomeDirectory();
743 workdir += fGridWorkingDir;
744 if (generate) {
745 TObjString *os;
746 if (!fInputFiles) {
747 Error("CreateJDL()", "Define some input files for your analysis.");
748 error = kTRUE;
749 }
750 // Compose list of input files
751 // Check if output files were defined
752 if (!fOutputFiles.Length()) {
753 Error("CreateJDL", "You must define at least one output file");
754 error = kTRUE;
755 }
756 // Check if an output directory was defined and valid
757 if (!fGridOutputDir.Length()) {
758 Error("CreateJDL", "You must define AliEn output directory");
759 error = kTRUE;
760 } else {
d2a409b2 761 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 762 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 763 if (gGrid->Mkdir(fGridOutputDir)) {
764 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
765 } else {
766 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 767 // error = kTRUE;
c57f56b7 768 }
769 }
770 gGrid->Cd(workdir);
771 }
772 // Exit if any error up to now
773 if (error) return kFALSE;
774 // Set JDL fields
775 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 776 fGridJDL->SetExecutable(fExecutable);
c57f56b7 777// fGridJDL->SetTTL((UInt_t)fTTL);
778 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
779 if (fMaxInitFailed > 0)
780 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
781 if (fSplitMaxInputFileNumber > 0)
782 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
783 if (fSplitMode.Length())
784 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
785// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
786 if (fAliROOTVersion.Length())
787 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
788 if (fROOTVersion.Length())
789 fGridJDL->AddToPackages("ROOT", fROOTVersion);
790 if (fAPIVersion.Length())
791 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
648174cf 792 if (!fExternalPackages.IsNull()) {
793 arr = fExternalPackages.Tokenize(" ");
794 TIter next(arr);
795 while ((os=(TObjString*)next())) {
796 TString pkgname = os->GetString();
797 Int_t index = pkgname.Index("::");
798 TString pkgversion = pkgname(index+2, pkgname.Length());
799 pkgname.Remove(index);
800 fGridJDL->AddToPackages(pkgname, pkgversion);
801 }
802 delete arr;
803 }
c57f56b7 804 fGridJDL->SetInputDataListFormat(fInputFormat);
805 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 806 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 807 TString analysisFile = fExecutable;
808 analysisFile.ReplaceAll(".sh", ".root");
809 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 810 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
811 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
812 if (fAdditionalLibs.Length()) {
813 arr = fAdditionalLibs.Tokenize(" ");
814 TIter next(arr);
815 while ((os=(TObjString*)next())) {
816 if (os->GetString().Contains(".so")) continue;
817 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
818 }
819 delete arr;
820 }
4e5c5506 821 if (fPackages) {
822 TIter next(fPackages);
823 TObject *obj;
824 while ((obj=next()))
825 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
826 }
c57f56b7 827 if (fOutputArchive.Length()) {
828 arr = fOutputArchive.Tokenize(" ");
829 TIter next(arr);
830 while ((os=(TObjString*)next()))
43da816a 831 if (!os->GetString().Contains("@") && fCloseSE.Length())
832 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
833 else
834 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 835 delete arr;
836 }
c57f56b7 837 arr = fOutputFiles.Tokenize(" ");
838 TIter next(arr);
43da816a 839 while ((os=(TObjString*)next())) {
e1eaf596 840 // Ignore ouputs in jdl that are also in outputarchive
841 TString sout = os->GetString();
842 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
843 if (fOutputArchive.Contains(sout)) continue;
43da816a 844 if (!os->GetString().Contains("@") && fCloseSE.Length())
845 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
846 else
847 fGridJDL->AddToOutputSandbox(os->GetString());
848 }
c57f56b7 849 delete arr;
850// fGridJDL->SetPrice((UInt_t)fPrice);
851 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 852 TString validationScript = fExecutable;
853 validationScript.ReplaceAll(".sh", "_validation.sh");
854 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 855 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 856 // Write a jdl with 2 input parameters: collection name and output dir name.
857 WriteJDL(copy);
c57f56b7 858 }
859 // Copy jdl to grid workspace
a8739e8a 860 if (copy) {
648174cf 861 if (TestBit(AliAnalysisGrid::kSubmit)) {
862 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
863 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
864 if (fProductionMode)
865 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
866 if (FileExists(locjdl)) gGrid->Rm(locjdl);
867 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
868 }
c57f56b7 869 if (fAdditionalLibs.Length()) {
e7c71df0 870 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 871 TObjString *os;
872 TIter next(arr);
873 while ((os=(TObjString*)next())) {
c57f56b7 874 if (os->GetString().Contains(".so")) continue;
4e5c5506 875 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 876 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
877 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
878 }
879 delete arr;
880 }
4e5c5506 881 if (fPackages) {
882 TIter next(fPackages);
883 TObject *obj;
884 while ((obj=next())) {
885 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
886 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
887 }
888 }
c57f56b7 889 }
890 return kTRUE;
891}
892
a8739e8a 893//______________________________________________________________________________
d2a409b2 894Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 895{
896// Writes one or more JDL's corresponding to findex. If findex is negative,
897// all run numbers are considered in one go (jdl). For non-negative indices
898// they correspond to the indices in the array fInputFiles.
899 if (!fInputFiles) return kFALSE;
900 TObjString *os;
a8739e8a 901 TString workdir = gGrid->GetHomeDirectory();
902 workdir += fGridWorkingDir;
d2a409b2 903
904 if (!fRunNumbers.Length() && !fRunRange[0]) {
905 // One jdl with no parameters in case input data is specified by name.
a8739e8a 906 TIter next(fInputFiles);
907 while ((os=(TObjString*)next()))
908 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
648174cf 909 if (!fOutputSingle.IsNull())
910 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
911 else
912 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 913 } else {
d2a409b2 914 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
915 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
648174cf 916 if (!fOutputSingle.IsNull())
917 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
e1eaf596 918 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 919 }
920
921
922 // Generate the JDL as a string
923 TString sjdl = fGridJDL->Generate();
924 Int_t index;
925 index = sjdl.Index("Executable");
926 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
927 index = sjdl.Index("Split ");
648174cf 928 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
a8739e8a 929 index = sjdl.Index("SplitMaxInputFileNumber");
930 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
931 index = sjdl.Index("InputDataCollection");
932 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 933 index = sjdl.Index("InputFile");
934 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
935 index = sjdl.Index("InputDataList ");
936 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
937 index = sjdl.Index("InputDataListFormat");
938 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
939 index = sjdl.Index("Price");
940 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
941 index = sjdl.Index("Requirements");
942 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
943 index = sjdl.Index("Packages");
944 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
945 index = sjdl.Index("User =");
946 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
947 index = sjdl.Index("TTL");
948 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
949 index = sjdl.Index("OutputFile");
950 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
951 index = sjdl.Index("OutputDir");
952 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
953 index = sjdl.Index("OutputArchive");
954 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
955 index = sjdl.Index("MaxInitFailed");
956 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
957 index = sjdl.Index("MasterResubmitThreshold");
958 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
959 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
960 index = sjdl.Index("Validationcommand");
961 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
962 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
963 sjdl.ReplaceAll("(member", "\n (member");
964 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
965 sjdl.ReplaceAll("{", "{\n ");
966 sjdl.ReplaceAll("};", "\n};");
967 sjdl.ReplaceAll("{\n \n", "{\n");
968 sjdl.ReplaceAll("\n\n", "\n");
969 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
970 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 971 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 972 index = sjdl.Index("JDLVariables");
973 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
974 // Write jdl to file
a8739e8a 975 ofstream out;
d2a409b2 976 out.open(fJDLName.Data(), ios::out);
a8739e8a 977 if (out.bad()) {
d2a409b2 978 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 979 return kFALSE;
980 }
981 out << sjdl << endl;
982
983 // Copy jdl to grid workspace
984 if (!copy) {
985 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
986 } else {
d2a409b2 987 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
988 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 989 if (fProductionMode)
b5fe9cba 990 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 991 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 992 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 993 }
994 return kTRUE;
995}
996
c57f56b7 997//______________________________________________________________________________
5513444a 998Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 999{
1000// Returns true if file exists.
5513444a 1001 if (!gGrid) return kFALSE;
c57f56b7 1002 TGridResult *res = gGrid->Ls(lfn);
1003 if (!res) return kFALSE;
1004 TMap *map = dynamic_cast<TMap*>(res->At(0));
1005 if (!map) {
1006 delete res;
1007 return kFALSE;
1008 }
1009 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1010 if (!objs || !objs->GetString().Length()) {
1011 delete res;
1012 return kFALSE;
1013 }
1014 delete res;
1015 return kTRUE;
1016}
1017
923e2ca5 1018//______________________________________________________________________________
1019Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1020{
1021// Returns true if directory exists. Can be also a path.
1022 if (!gGrid) return kFALSE;
1023 // Check if dirname is a path
1024 TString dirstripped = dirname;
1025 dirstripped = dirstripped.Strip();
1026 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1027 TString dir = gSystem->BaseName(dirstripped);
1028 dir += "/";
1029 TString path = gSystem->DirName(dirstripped);
1030 TGridResult *res = gGrid->Ls(path, "-F");
1031 if (!res) return kFALSE;
1032 TIter next(res);
1033 TMap *map;
1034 TObject *obj;
1035 while ((map=dynamic_cast<TMap*>(next()))) {
1036 obj = map->GetValue("name");
1037 if (!obj) break;
1038 if (dir == obj->GetName()) {
1039 delete res;
1040 return kTRUE;
1041 }
1042 }
1043 delete res;
1044 return kFALSE;
1045}
1046
c57f56b7 1047//______________________________________________________________________________
1048void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1049{
1050// Check input data type.
1051 is_collection = kFALSE;
1052 is_xml = kFALSE;
1053 use_tags = kFALSE;
1054 if (!gGrid) {
1055 Error("CheckDataType", "No connection to grid");
1056 return;
1057 }
1058 is_collection = IsCollection(lfn);
1059 TString msg = "\n##### file: ";
1060 msg += lfn;
1061 if (is_collection) {
1062 msg += " type: raw_collection;";
1063 // special treatment for collections
1064 is_xml = kFALSE;
1065 // check for tag files in the collection
1066 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1067 if (!res) {
1068 msg += " using_tags: No (unknown)";
1069 Info("CheckDataType", msg.Data());
1070 return;
1071 }
1072 const char* typeStr = res->GetKey(0, "origLFN");
1073 if (!typeStr || !strlen(typeStr)) {
1074 msg += " using_tags: No (unknown)";
1075 Info("CheckDataType", msg.Data());
1076 return;
1077 }
1078 TString file = typeStr;
1079 use_tags = file.Contains(".tag");
1080 if (use_tags) msg += " using_tags: Yes";
1081 else msg += " using_tags: No";
1082 Info("CheckDataType", msg.Data());
1083 return;
1084 }
1085 TString slfn(lfn);
1086 slfn.ToLower();
1087 is_xml = slfn.Contains(".xml");
1088 if (is_xml) {
1089 // Open xml collection and check if there are tag files inside
1090 msg += " type: xml_collection;";
1091 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1092 if (!coll) {
1093 msg += " using_tags: No (unknown)";
1094 Info("CheckDataType", msg.Data());
1095 return;
1096 }
1097 TMap *map = coll->Next();
1098 if (!map) {
1099 msg += " using_tags: No (unknown)";
1100 Info("CheckDataType", msg.Data());
1101 return;
1102 }
1103 map = (TMap*)map->GetValue("");
1104 TString file;
1105 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1106 use_tags = file.Contains(".tag");
1107 delete coll;
1108 if (use_tags) msg += " using_tags: Yes";
1109 else msg += " using_tags: No";
1110 Info("CheckDataType", msg.Data());
1111 return;
1112 }
1113 use_tags = slfn.Contains(".tag");
1114 if (slfn.Contains(".root")) msg += " type: root file;";
1115 else msg += " type: unhnown file;";
1116 if (use_tags) msg += " using_tags: Yes";
1117 else msg += " using_tags: No";
1118 Info("CheckDataType", msg.Data());
1119}
1120
4e5c5506 1121//______________________________________________________________________________
1122void AliAnalysisAlien::EnablePackage(const char *package)
1123{
1124// Enables a par file supposed to exist in the current directory.
1125 TString pkg(package);
1126 pkg.ReplaceAll(".par", "");
1127 pkg += ".par";
1128 if (gSystem->AccessPathName(pkg)) {
1129 Error("EnablePackage", "Package %s not found", pkg.Data());
1130 return;
1131 }
1132 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1133 Info("EnablePackage", "AliEn plugin will use .par packages");
1134 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1135 if (!fPackages) {
1136 fPackages = new TObjArray();
1137 fPackages->SetOwner();
1138 }
1139 fPackages->Add(new TObjString(pkg));
1140}
1141
d2a409b2 1142//______________________________________________________________________________
1143const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1144{
1145// Get job status for all jobs with jobid>jobidstart.
1146 static char mstatus[20];
1147 mstatus[0] = '\0';
1148 nrunning = 0;
1149 nwaiting = 0;
1150 nerror = 0;
1151 ndone = 0;
1152 TGridJobStatusList *list = gGrid->Ps("");
1153 if (!list) return mstatus;
1154 Int_t nentries = list->GetSize();
1155 TGridJobStatus *status;
1156 Int_t pid;
1157 for (Int_t ijob=0; ijob<nentries; ijob++) {
1158 status = (TGridJobStatus *)list->At(ijob);
1159 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1160 if (pid<jobidstart) continue;
1161 if (pid == lastid) {
1162 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1163 }
1164 switch (status->GetStatus()) {
1165 case TGridJobStatus::kWAITING:
1166 nwaiting++; break;
1167 case TGridJobStatus::kRUNNING:
1168 nrunning++; break;
1169 case TGridJobStatus::kABORTED:
1170 case TGridJobStatus::kFAIL:
1171 case TGridJobStatus::kUNKNOWN:
1172 nerror++; break;
1173 case TGridJobStatus::kDONE:
1174 ndone++;
1175 }
1176 }
1177 list->Delete();
1178 delete list;
1179 return mstatus;
1180}
1181
c57f56b7 1182//______________________________________________________________________________
1183Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1184{
1185// Returns true if file is a collection. Functionality duplicated from
1186// TAlien::Type() because we don't want to directly depend on TAlien.
1187 if (!gGrid) {
1188 Error("IsCollection", "No connection to grid");
1189 return kFALSE;
1190 }
1191 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1192 if (!res) return kFALSE;
1193 const char* typeStr = res->GetKey(0, "type");
1194 if (!typeStr || !strlen(typeStr)) return kFALSE;
1195 if (!strcmp(typeStr, "collection")) return kTRUE;
1196 delete res;
1197 return kFALSE;
1198}
1199
16a4353c 1200//______________________________________________________________________________
1201void AliAnalysisAlien::Print(Option_t *) const
1202{
1203// Print current plugin settings.
84fcd93f 1204 printf("### AliEn analysis plugin current settings ###\n");
1205 printf("= Production mode:______________________________ %d\n", fProductionMode);
1206 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1207 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1208 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1209 if (fUser.Length())
84fcd93f 1210 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1211 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1212 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1213 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1214 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1215 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1216 if (fRunNumbers.Length())
84fcd93f 1217 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1218 if (fRunRange[0])
84fcd93f 1219 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1220 if (!fRunRange[0] && !fRunNumbers.Length()) {
1221 TIter next(fInputFiles);
1222 TObject *obj;
1223 TString list;
1224 while ((obj=next())) list += obj->GetName();
84fcd93f 1225 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1226 }
1227 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1228 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1229 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1230 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1231 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1232 printf("=====================================================================\n");
1233 printf("= Job price: ___________________________________ %d\n", fPrice);
1234 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1235 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1236 if (fMaxInitFailed>0)
84fcd93f 1237 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1238 if (fMasterResubmitThreshold>0)
84fcd93f 1239 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1240 if (fNrunsPerMaster>0)
84fcd93f 1241 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1242 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1243 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
16a4353c 1244 if (fArguments.Length())
84fcd93f 1245 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1246 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1247 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1248 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1249 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1250 if (fDatasetName)
84fcd93f 1251 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1252 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1253 if (fIncludePath.Data())
84fcd93f 1254 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1255 if (fCloseSE.Length())
84fcd93f 1256 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1257 if (fFriendChainName.Length())
84fcd93f 1258 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 TString list;
1263 while ((obj=next())) list += obj->GetName();
84fcd93f 1264 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1265 }
1266}
1267
c57f56b7 1268//______________________________________________________________________________
1269void AliAnalysisAlien::SetDefaults()
1270{
1271// Set default values for everything. What cannot be filled will be left empty.
1272 if (fGridJDL) delete fGridJDL;
1273 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1274 fPrice = 1;
1275 fTTL = 30000;
1276 fSplitMaxInputFileNumber = 100;
1277 fMaxInitFailed = 0;
1278 fMasterResubmitThreshold = 0;
bb885a9e 1279 fNtestFiles = 10;
a8739e8a 1280 fRunRange[0] = 0;
1281 fRunRange[1] = 0;
319593fb 1282 fNrunsPerMaster = 1;
16a4353c 1283 fMaxMergeFiles = 100;
c57f56b7 1284 fRunNumbers = "";
1285 fExecutable = "analysis.sh";
0a1c1f7f 1286 fExecutableCommand = "root -b -q";
c57f56b7 1287 fArguments = "";
1288 fAnalysisMacro = "myAnalysis.C";
1289 fAnalysisSource = "";
1290 fAdditionalLibs = "";
1291 fSplitMode = "se";
1292 fAPIVersion = "";
1293 fROOTVersion = "";
1294 fAliROOTVersion = "";
1295 fUser = ""; // Your alien user name
1296 fGridWorkingDir = "";
1297 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1298 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1299 fFriendChainName = "";
c57f56b7 1300 fGridOutputDir = "output";
1301 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1302 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1303 fInputFormat = "xml-single";
1304 fJDLName = "analysis.jdl";
c6cb3634 1305 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1306 fMergeExcludes = "";
c57f56b7 1307}
1308
1309//______________________________________________________________________________
1310Bool_t AliAnalysisAlien::MergeOutputs()
1311{
1312// Merge analysis outputs existing in the AliEn space.
1313 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1314 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1315 if (!Connect()) {
1316 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1317 return kFALSE;
1318 }
1319 // Get the output path
d2a409b2 1320 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1321 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1322 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1323 return kFALSE;
1324 }
1325 if (!fOutputFiles.Length()) {
1326 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1327 return kFALSE;
1328 }
1329 TObjArray *list = fOutputFiles.Tokenize(" ");
1330 TIter next(list);
1331 TObjString *str;
1332 TString command;
1333 TString output_file;
16a4353c 1334 TString output_chunk;
1335 TString previous_chunk;
1336 Int_t count_chunk = 0;
1337 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1338 Bool_t merged = kTRUE;
1339 while((str=(TObjString*)next())) {
1340 output_file = str->GetString();
1341 Int_t index = output_file.Index("@");
1342 if (index > 0) output_file.Remove(index);
319593fb 1343 // Skip already merged outputs
1344 if (!gSystem->AccessPathName(output_file)) {
1345 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1346 continue;
1347 }
bb885a9e 1348 if (fMergeExcludes.Length() &&
1349 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1350 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1351 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1352 printf("command: %s\n", command.Data());
c57f56b7 1353 TGridResult *res = gGrid->Command(command);
1354 if (!res) continue;
1355 TFileMerger *fm = 0;
1356 TIter nextmap(res);
ab254fd1 1357 TMap *map = 0;
16a4353c 1358 previous_chunk = "";
1359 count_chunk = 0;
319593fb 1360 // Check if there is a merge operation to resume
1361 output_chunk = output_file;
1362 output_chunk.ReplaceAll(".root", "_*.root");
1363 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1364 while (1) {
1365 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1366 if (!map) {
1367 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1368 delete res;
1369 return kFALSE;
1370 }
1371 output_chunk = output_file;
1372 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1373 printf("%s\n", output_chunk.Data());
319593fb 1374 count_chunk++;
1375 if (gSystem->AccessPathName(output_chunk)) continue;
1376 // Merged file with chunks up to <count_chunk> found
84fcd93f 1377 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1378 previous_chunk = output_chunk;
1379 break;
1380 }
1381 }
16a4353c 1382 count_zero = fMaxMergeFiles;
c57f56b7 1383 while ((map=(TMap*)nextmap())) {
16a4353c 1384 // Loop 'find' results and get next LFN
1385 if (count_zero == fMaxMergeFiles) {
1386 // First file in chunk - create file merger and add previous chunk if any.
1387 fm = new TFileMerger(kFALSE);
1388 fm->SetFastMethod(kTRUE);
1389 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1390 output_chunk = output_file;
1391 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1392 }
1393 // If last file found, put merged results in the output file
1394 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1395 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1396 if (!objs || !objs->GetString().Length()) {
16a4353c 1397 // Nothing found - skip this output
c57f56b7 1398 delete res;
16a4353c 1399 delete fm;
1400 break;
1401 }
1402 // Add file to be merged and decrement chunk counter.
1403 fm->AddFile(objs->GetString());
1404 count_zero--;
1405 if (count_zero==0 || map == res->Last()) {
1406 fm->OutputFile(output_chunk);
1407 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1408 // Nothing found - skip this output
1409 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1410 delete res;
1411 delete fm;
1412 break;
1413 }
1414 // Merge the outputs, then go to next chunk
1415 if (!fm->Merge()) {
1416 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1417 delete res;
1418 delete fm;
1419 merged = kFALSE;
1420 break;
1421 } else {
1422 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1423 gSystem->Unlink(previous_chunk);
16a4353c 1424 }
1425 if (map == res->Last()) {
1426 delete res;
1427 delete fm;
1428 break;
1429 }
1430 count_chunk++;
1431 count_zero = fMaxMergeFiles;
1432 previous_chunk = output_chunk;
c57f56b7 1433 }
c57f56b7 1434 }
c57f56b7 1435 }
1436 if (!merged) {
1437 Error("MergeOutputs", "Terminate() will NOT be executed");
1438 }
1439 return merged;
1440}
1441
bb885a9e 1442//______________________________________________________________________________
1443void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1444{
1445// Use the output files connected to output containers from the analysis manager
1446// rather than the files defined by SetOutputFiles
1447 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1448 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1449 analysis manager");
1450 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1451}
1452
5513444a 1453//______________________________________________________________________________
1454Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1455{
1456// Start remote grid analysis.
1457
43da816a 1458 // Check if output files have to be taken from the analysis manager
1459 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1460 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1461 if (!mgr || !mgr->IsInitialized()) {
1462 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1463 return kFALSE;
43da816a 1464 }
1465 fOutputFiles = "";
1466 TIter next(mgr->GetOutputs());
1467 AliAnalysisDataContainer *output;
1468 while ((output=(AliAnalysisDataContainer*)next())) {
1469 const char *filename = output->GetFileName();
1470 if (!(strcmp(filename, "default"))) {
1471 if (!mgr->GetOutputEventHandler()) continue;
1472 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1473 }
84fcd93f 1474 if (fOutputFiles.Contains(filename)) continue;
43da816a 1475 if (fOutputFiles.Length()) fOutputFiles += " ";
1476 fOutputFiles += filename;
1477 }
c07b9ce2 1478 // Add extra files registered to the analysis manager
1479 if (mgr->GetExtraFiles().Length()) {
1480 if (fOutputFiles.Length()) fOutputFiles += " ";
1481 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1482 }
43da816a 1483 }
f7b1cbc2 1484// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1485 if (TestBit(AliAnalysisGrid::kOffline)) {
1486 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1487 \n there nor any job run. You can revise the JDL and analysis \
1488 \n macro then run the same in \"submit\" mode.");
1489 } else if (TestBit(AliAnalysisGrid::kTest)) {
1490 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1491 \n dataset.");
1492 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1493 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1494 \n space and job submitted.");
1495 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1496 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1497 return kTRUE;
c57f56b7 1498 } else {
1499 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1500 }
1501
1502 if (!Connect()) {
1503 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1504 return kFALSE;
16a4353c 1505 }
1506 Print();
c57f56b7 1507 if (!CheckInputData()) {
1508 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1509 return kFALSE;
c57f56b7 1510 }
1511 CreateDataset(fDataPattern);
1512 WriteAnalysisFile();
1513 WriteAnalysisMacro();
1514 WriteExecutable();
1515 WriteValidationScript();
5513444a 1516 if (!CreateJDL()) return kFALSE;
1517 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1518 if (TestBit(AliAnalysisGrid::kTest)) {
1519 // Locally testing the analysis
1520 Info("StartAnalysis", "\n_______________________________________________________________________ \
1521 \n Running analysis script in a daughter shell as on a worker node \
1522 \n_______________________________________________________________________");
1523 TObjArray *list = fOutputFiles.Tokenize(" ");
1524 TIter next(list);
1525 TObjString *str;
1526 TString output_file;
1527 while((str=(TObjString*)next())) {
1528 output_file = str->GetString();
1529 Int_t index = output_file.Index("@");
1530 if (index > 0) output_file.Remove(index);
43da816a 1531 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1532 }
1533 delete list;
1534 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1535 TString validationScript = fExecutable;
1536 validationScript.ReplaceAll(".sh", "_validation.sh");
1537 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1538// gSystem->Exec("cat stdout");
5513444a 1539 return kFALSE;
c57f56b7 1540 }
5513444a 1541 // Check if submitting is managed by LPM manager
a3e84053 1542 if (fProductionMode) {
5513444a 1543 TString prodfile = fJDLName;
1544 prodfile.ReplaceAll(".jdl", ".prod");
1545 WriteProductionFile(prodfile);
1546 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1547 return kFALSE;
1548 }
a8739e8a 1549 // Submit AliEn job(s)
d2a409b2 1550 gGrid->Cd(fGridOutputDir);
a8739e8a 1551 TGridResult *res;
c57f56b7 1552 TString jobID = "";
d2a409b2 1553 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1554 // Submit a given xml or a set of runs
a8739e8a 1555 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1556 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1557 if (res) {
1558 const char *cjobId = res->GetKey(0,"jobId");
1559 if (!cjobId) {
1560 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1561 return kFALSE;
a8739e8a 1562 } else {
1563 Info("StartAnalysis", "\n_______________________________________________________________________ \
1564 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1565 \n_______________________________________________________________________",
1566 fJDLName.Data(), cjobId);
1567 jobID = cjobId;
1568 }
1569 delete res;
1570 }
1571 } else {
d2a409b2 1572 // Submit for a range of enumeration of runs.
1573 Submit();
c57f56b7 1574 }
a8739e8a 1575
c57f56b7 1576 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1577 \n You may exit at any time and terminate the job later using the option <terminate> \
1578 \n ##################################################################################", jobID.Data());
bb885a9e 1579 gSystem->Exec("aliensh");
5513444a 1580 return kTRUE;
c57f56b7 1581}
1582
d2a409b2 1583//______________________________________________________________________________
1584void AliAnalysisAlien::Submit()
1585{
1586// Submit all master jobs.
1587 Int_t nmasterjobs = fInputFiles->GetEntries();
1588 Long_t tshoot = gSystem->Now();
1589 if (!fNsubmitted) SubmitNext();
1590 while (fNsubmitted < nmasterjobs) {
1591 Long_t now = gSystem->Now();
1592 if ((now-tshoot)>30000) {
1593 tshoot = now;
1594 SubmitNext();
1595 }
1596 }
1597}
1598
1599//______________________________________________________________________________
1600void AliAnalysisAlien::SubmitNext()
1601{
1602// Submit next bunch of master jobs if the queue is free.
1603 static Bool_t iscalled = kFALSE;
1604 static Int_t firstmaster = 0;
1605 static Int_t lastmaster = 0;
1606 static Int_t npermaster = 0;
1607 if (iscalled) return;
1608 iscalled = kTRUE;
1609 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1610 Int_t ntosubmit = 0;
1611 TGridResult *res;
1612 TString jobID = "";
1613 if (!fNsubmitted) ntosubmit = 1;
1614 else {
1615 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1616 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1617 // If last master not split, just return
1618 if (status != "SPLIT") {iscalled = kFALSE; return;}
1619 // No more than 100 waiting jobs
1620 if (nwaiting>100) {iscalled = kFALSE; return;}
1621 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1622 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1623 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1624 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1625 }
1626 Int_t nmasterjobs = fInputFiles->GetEntries();
1627 for (Int_t i=0; i<ntosubmit; i++) {
1628 // Submit for a range of enumeration of runs.
1629 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1630 TString query;
1631 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1632 printf("********* %s\n",query.Data());
d2a409b2 1633 res = gGrid->Command(query);
1634 if (res) {
98ca124f 1635 TString cjobId1 = res->GetKey(0,"jobId");
1636 if (!cjobId1.Length()) {
d2a409b2 1637 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1638 iscalled = kFALSE;
1639 return;
1640 } else {
1641 Info("StartAnalysis", "\n_______________________________________________________________________ \
1642 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1643 \n_______________________________________________________________________",
98ca124f 1644 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1645 jobID += cjobId1;
1646 jobID += " ";
98ca124f 1647 lastmaster = cjobId1.Atoi();
d2a409b2 1648 if (!firstmaster) firstmaster = lastmaster;
1649 fNsubmitted++;
1650 }
1651 delete res;
1652 }
1653 }
1654 iscalled = kFALSE;
1655}
1656
c57f56b7 1657//______________________________________________________________________________
1658void AliAnalysisAlien::WriteAnalysisFile()
1659{
f10e8481 1660// Write current analysis manager into the file <analysisFile>
1661 TString analysisFile = fExecutable;
1662 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1663 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1664 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1665 if (!mgr || !mgr->IsInitialized()) {
1666 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1667 return;
1668 }
1669 // Check analysis type
1670 TObject *handler;
1671 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1672 handler = (TObject*)mgr->GetInputEventHandler();
1673 if (handler) {
1674 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1675 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1676 }
1677 TDirectory *cdir = gDirectory;
f10e8481 1678 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1679 if (file) {
1680 mgr->Write();
1681 delete file;
1682 }
1683 if (cdir) cdir->cd();
f10e8481 1684 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1685 }
1686 Bool_t copy = kTRUE;
1687 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1688 if (copy) {
1689 CdWork();
1690 TString workdir = gGrid->GetHomeDirectory();
1691 workdir += fGridWorkingDir;
f10e8481 1692 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1693 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1694 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1695 }
1696}
1697
1698//______________________________________________________________________________
1699void AliAnalysisAlien::WriteAnalysisMacro()
1700{
1701// Write the analysis macro that will steer the analysis in grid mode.
1702 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1703 ofstream out;
1704 out.open(fAnalysisMacro.Data(), ios::out);
1705 if (!out.good()) {
1706 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1707 return;
1708 }
1709 TString func = fAnalysisMacro;
1710 TString type = "ESD";
1711 TString comment = "// Analysis using ";
1712 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1713 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1714 type = "AOD";
1715 comment += "AOD";
1716 }
0df6ccf2 1717 if (type!="AOD" && fFriendChainName!="") {
1718 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1719 return;
1720 }
c57f56b7 1721 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1722 else comment += " data";
1723 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1724 func.ReplaceAll(".C", "");
1725 out << "void " << func.Data() << "()" << endl;
1726 out << "{" << endl;
1727 out << comment.Data() << endl;
1728 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1729 out << " TStopwatch timer;" << endl;
1730 out << " timer.Start();" << endl << endl;
c57f56b7 1731 out << "// load base root libraries" << endl;
1732 out << " gSystem->Load(\"libTree\");" << endl;
1733 out << " gSystem->Load(\"libGeom\");" << endl;
1734 out << " gSystem->Load(\"libVMC\");" << endl;
1735 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
648174cf 1736 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
57377eb5 1737 out << "// Load analysis framework libraries" << endl;
4e5c5506 1738 if (!fPackages) {
4e5c5506 1739 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1740 out << " gSystem->Load(\"libESD\");" << endl;
1741 out << " gSystem->Load(\"libAOD\");" << endl;
1742 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1743 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1744 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1745 } else {
4e5c5506 1746 TIter next(fPackages);
1747 TObject *obj;
57377eb5 1748 TString pkgname;
1749 Bool_t hasSTEERBase = kFALSE;
1750 Bool_t hasESD = kFALSE;
1751 Bool_t hasAOD = kFALSE;
1752 Bool_t hasANALYSIS = kFALSE;
1753 Bool_t hasANALYSISalice = kFALSE;
1754 Bool_t hasCORRFW = kFALSE;
1755 while ((obj=next())) {
1756 pkgname = obj->GetName();
4478e6f1 1757 if (pkgname == "STEERBase" ||
1758 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1759 if (pkgname == "ESD" ||
1760 pkgname == "ESD.par") hasESD = kTRUE;
1761 if (pkgname == "AOD" ||
1762 pkgname == "AOD.par") hasAOD = kTRUE;
1763 if (pkgname == "ANALYSIS" ||
1764 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1765 if (pkgname == "ANALYSISalice" ||
1766 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1767 if (pkgname == "CORRFW" ||
1768 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1769 }
1770 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1771 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1772 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1773 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1774 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1775 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1776 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1777 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1778 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1779 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1780 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1781 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1782 out << "// Compile other par packages" << endl;
1783 next.Reset();
fcc9bb6f 1784 while ((obj=next())) {
1785 pkgname = obj->GetName();
4478e6f1 1786 if (pkgname == "STEERBase" ||
1787 pkgname == "STEERBase.par" ||
1788 pkgname == "ESD" ||
1789 pkgname == "ESD.par" ||
1790 pkgname == "AOD" ||
1791 pkgname == "AOD.par" ||
1792 pkgname == "ANALYSIS" ||
1793 pkgname == "ANALYSIS.par" ||
1794 pkgname == "ANALYSISalice" ||
1795 pkgname == "ANALYSISalice.par" ||
1796 pkgname == "CORRFW" ||
1797 pkgname == "CORRFW.par") continue;
4e5c5506 1798 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1799 }
4e5c5506 1800 }
43da816a 1801 out << "// include path" << endl;
1802 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1803 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1804 if (fAdditionalLibs.Length()) {
1805 out << "// Add aditional AliRoot libraries" << endl;
1806 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1807 TIter next(list);
1808 TObjString *str;
1809 while((str=(TObjString*)next())) {
1810 if (str->GetString().Contains(".so"))
1811 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1812 }
1813 if (list) delete list;
1814 }
1815 out << endl;
c57f56b7 1816 out << "// analysis source to be compiled at runtime (if any)" << endl;
1817 if (fAnalysisSource.Length()) {
1818 TObjArray *list = fAnalysisSource.Tokenize(" ");
1819 TIter next(list);
1820 TObjString *str;
1821 while((str=(TObjString*)next())) {
1822 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1823 }
1824 if (list) delete list;
1825 }
1826 out << endl;
1827 out << "// connect to AliEn and make the chain" << endl;
1828 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1829 if (IsUsingTags()) {
1830 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1831 } else {
76535c98 1832 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1833 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1834 } else {
1835 out << " // Check if the macro to create the chain was provided" << endl;
1836 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1837 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1838 out << " return;" << endl;
1839 out << " }" << endl;
1840 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1841 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1842 }
c57f56b7 1843 }
1844 out << "// read the analysis manager from file" << endl;
f10e8481 1845 TString analysisFile = fExecutable;
1846 analysisFile.ReplaceAll(".sh", ".root");
1847 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1848 out << " if (!file) return;" << endl;
1849 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1850 out << " AliAnalysisManager *mgr = 0;" << endl;
1851 out << " TKey *key;" << endl;
1852 out << " while ((key=(TKey*)nextkey())) {" << endl;
1853 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1854 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1855 out << " };" << endl;
1856 out << " if (!mgr) {" << endl;
f10e8481 1857 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1858 out << " return;" << endl;
1859 out << " }" << endl << endl;
1860 out << " mgr->PrintStatus();" << endl;
1861 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1862 out << " timer.Stop();" << endl;
1863 out << " timer.Print();" << endl;
c57f56b7 1864 out << "}" << endl << endl;
1865 if (IsUsingTags()) {
1866 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1867 out << "{" << endl;
1868 out << "// Create a chain using tags from the xml file." << endl;
1869 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1870 out << " if (!coll) {" << endl;
1871 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1872 out << " return NULL;" << endl;
1873 out << " }" << endl;
1874 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1875 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1876 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1877 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1878 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1879 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1880 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1881 out << " // Check if the cuts configuration file was provided" << endl;
1882 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1883 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1884 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1885 out << " }" << endl;
0df6ccf2 1886 if (fFriendChainName=="") {
1887 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1888 } else {
1889 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1890 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1891 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1892 }
c57f56b7 1893 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1894 out << " chain->ls();" << endl;
1895 out << " return chain;" << endl;
fcc9bb6f 1896 out << "}" << endl << endl;
c57f56b7 1897 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1898 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1899 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1900 msg += " AliLHCTagCuts *lhcCuts,\n";
1901 msg += " AliDetectorTagCuts *detCuts,\n";
1902 msg += " AliEventTagCuts *evCuts)";
1903 Info("WriteAnalysisMacro", msg.Data());
1904 }
0df6ccf2 1905 }
1906 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1907 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1908 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1909 out << "{" << endl;
1910 out << "// Create a chain using url's from xml file" << endl;
1911 out << " TString treename = type;" << endl;
1912 out << " treename.ToLower();" << endl;
1913 out << " treename += \"Tree\";" << endl;
e02fee64 1914 out << " printf(\"***************************************\\n\");" << endl;
1915 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1916 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1917 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1918 out << " if (!coll) {" << endl;
1919 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1920 out << " return NULL;" << endl;
1921 out << " }" << endl;
1922 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1923 if(fFriendChainName!="") {
1924 out << " TChain *chainFriend = new TChain(treename);" << endl;
1925 }
c57f56b7 1926 out << " coll->Reset();" << endl;
0df6ccf2 1927 out << " while (coll->Next()) {" << endl;
1928 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1929 if(fFriendChainName!="") {
1930 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1931 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1932 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1933 out << " chainFriend->Add(fileFriend.Data());" << endl;
1934 }
1935 out << " }" << endl;
c57f56b7 1936 out << " if (!chain->GetNtrees()) {" << endl;
1937 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1938 out << " return NULL;" << endl;
1939 out << " }" << endl;
0df6ccf2 1940 if(fFriendChainName!="") {
1941 out << " chain->AddFriend(chainFriend);" << endl;
1942 }
c57f56b7 1943 out << " return chain;" << endl;
fcc9bb6f 1944 out << "}" << endl << endl;
c57f56b7 1945 }
4e5c5506 1946 if (fPackages) {
fcc9bb6f 1947 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1948 out << "Bool_t SetupPar(const char *package) {" << endl;
1949 out << "// Compile the package and set it up." << endl;
1950 out << " TString pkgdir = package;" << endl;
1951 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1952 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1953 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1954 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1955 out << " // Check for BUILD.sh and execute" << endl;
1956 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1957 out << " printf(\"*******************************\\n\");" << endl;
1958 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1959 out << " printf(\"*******************************\\n\");" << endl;
1960 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1961 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1962 out << " gSystem->ChangeDirectory(cdir);" << endl;
1963 out << " return kFALSE;" << endl;
1964 out << " }" << endl;
1965 out << " } else {" << endl;
fcc9bb6f 1966 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1967 out << " gSystem->ChangeDirectory(cdir);" << endl;
1968 out << " return kFALSE;" << endl;
1969 out << " }" << endl;
1970 out << " // Check for SETUP.C and execute" << endl;
1971 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1972 out << " printf(\"*******************************\\n\");" << endl;
1973 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1974 out << " printf(\"*******************************\\n\");" << endl;
1975 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1976 out << " } else {" << endl;
fcc9bb6f 1977 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1978 out << " gSystem->ChangeDirectory(cdir);" << endl;
1979 out << " return kFALSE;" << endl;
1980 out << " }" << endl;
1981 out << " // Restore original workdir" << endl;
1982 out << " gSystem->ChangeDirectory(cdir);" << endl;
1983 out << " return kTRUE;" << endl;
1984 out << "}" << endl;
1985 }
c57f56b7 1986 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1987 }
1988 Bool_t copy = kTRUE;
1989 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1990 if (copy) {
1991 CdWork();
1992 TString workdir = gGrid->GetHomeDirectory();
1993 workdir += fGridWorkingDir;
1994 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1995 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1996 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1997 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1998 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1999 }
2000 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2001 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2002 }
2003}
2004
2005//______________________________________________________________________________
2006void AliAnalysisAlien::WriteExecutable()
2007{
2008// Generate the alien executable script.
2009 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2010 ofstream out;
2011 out.open(fExecutable.Data(), ios::out);
2012 if (out.bad()) {
5513444a 2013 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 2014 return;
2015 }
2016 out << "#!/bin/bash" << endl;
c57f56b7 2017 out << "echo \"=========================================\"" << endl;
2018 out << "echo \"############## PATH : ##############\"" << endl;
2019 out << "echo $PATH" << endl;
2020 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2021 out << "echo $LD_LIBRARY_PATH" << endl;
2022 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2023 out << "echo $ROOTSYS" << endl;
2024 out << "echo \"############## which root : ##############\"" << endl;
2025 out << "which root" << endl;
2026 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2027 out << "echo $ALICE_ROOT" << endl;
2028 out << "echo \"############## which aliroot : ##############\"" << endl;
2029 out << "which aliroot" << endl;
2030 out << "echo \"=========================================\"" << endl << endl;
74a53467 2031 // Make sure we can properly compile par files
2032 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 2033 out << fExecutableCommand << " ";
c57f56b7 2034 out << fAnalysisMacro.Data() << endl << endl;
2035 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
2036 }
2037 Bool_t copy = kTRUE;
2038 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2039 if (copy) {
2040 CdWork();
2041 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2042 TString bindir = Form("%s/bin", workdir.Data());
2043 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2044 workdir += fGridWorkingDir;
2045 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2046 if (FileExists(executable)) gGrid->Rm(executable);
2047 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2048 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2049 }
2050}
2051
2052//______________________________________________________________________________
5513444a 2053void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2054{
2055// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2056// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2057// Next lines: full_path_to_dataset XXX (XXX is a string)
2058// To submit, one has to: submit jdl XXX for all lines
2059 ofstream out;
2060 out.open(filename, ios::out);
2061 if (out.bad()) {
2062 Error("WriteProductionFile", "Bad file name: %s", filename);
2063 return;
2064 }
2065 TString workdir = gGrid->GetHomeDirectory();
2066 workdir += fGridWorkingDir;
f5e8c702 2067 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2068 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2069 out << locjdl << " " << njobspermaster << endl;
5513444a 2070 Int_t nmasterjobs = fInputFiles->GetEntries();
2071 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2072 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2073 }
2074 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2075 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2076}
2077
2078//______________________________________________________________________________
c57f56b7 2079void AliAnalysisAlien::WriteValidationScript()
2080{
2081// Generate the alien validation script.
2082 // Generate the validation script
2083 TObjString *os;
0d5d317c 2084 TString validationScript = fExecutable;
2085 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2086 if (!Connect()) {
2087 Error("WriteValidationScript", "Alien connection required");
2088 return;
2089 }
2090 TString out_stream = "";
2091 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2092 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2093 ofstream out;
0d5d317c 2094 out.open(validationScript, ios::out);
c57f56b7 2095 out << "#!/bin/bash" << endl;
2096 out << "##################################################" << endl;
2097 out << "validateout=`dirname $0`" << endl;
2098 out << "validatetime=`date`" << endl;
2099 out << "validated=\"0\";" << endl;
2100 out << "error=0" << endl;
2101 out << "if [ -z $validateout ]" << endl;
2102 out << "then" << endl;
2103 out << " validateout=\".\"" << endl;
2104 out << "fi" << endl << endl;
2105 out << "cd $validateout;" << endl;
2106 out << "validateworkdir=`pwd`;" << endl << endl;
2107 out << "echo \"*******************************************************\"" << out_stream << endl;
2108 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2109 out << "" << endl;
2110 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2111 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2112 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2113 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2114 out << "ls -la ./" << out_stream << endl;
2115 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2116 out << "##################################################" << endl;
ebec370a 2117
2118 out << "" << endl;
2119 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2120 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2121 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2122 out << "" << endl;
2123
2124 out << "if [ ! -f stderr ] ; then" << endl;
2125 out << " error=1" << endl;
2126 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2127 out << " echo \"Error = $error\" " << out_stream << endl;
2128 out << "fi" << endl;
2129
2130 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2131 out << " error=1" << endl;
2132 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2133 out << " echo \"$parArch\" " << out_stream << endl;
2134 out << " echo \"Error = $error\" " << out_stream << endl;
2135 out << "fi" << endl;
2136
2137 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2138 out << " error=1" << endl;
2139 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2140 out << " echo \"$segViol\" " << out_stream << endl;
2141 out << " echo \"Error = $error\" " << out_stream << endl;
2142 out << "fi" << endl;
2143
2144 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2145 out << " error=1" << endl;
2146 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2147 out << " echo \"$segFault\" " << out_stream << endl;
2148 out << " echo \"Error = $error\" " << out_stream << endl;
2149 out << "fi" << endl;
2150
2151 // Part dedicated to the specific analyses running into the train
2152
c57f56b7 2153 TObjArray *arr = fOutputFiles.Tokenize(" ");
2154 TIter next1(arr);
2155 TString output_file;
2156 while ((os=(TObjString*)next1())) {
2157 output_file = os->GetString();
2158 Int_t index = output_file.Index("@");
2159 if (index > 0) output_file.Remove(index);
2160 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2161 out << " error=1" << endl;
2162 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2163 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2164 out << "fi" << endl;
2165 }
2166 delete arr;
923e2ca5 2167 out << "if ! [ -f outputs_valid ] ; then" << endl;
2168 out << " error=1" << endl;
2169 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2170 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2171 out << "fi" << endl;
2172
c57f56b7 2173 out << "if [ $error = 0 ] ; then" << endl;
2174 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2175 out << "fi" << endl;
2176
2177 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2178 out << "echo \"*******************************************************\"" << out_stream << endl;
2179 out << "cd -" << endl;
2180 out << "exit $error" << endl;
2181 }
2182 Bool_t copy = kTRUE;
2183 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2184 if (copy) {
2185 CdWork();
2186 TString workdir = gGrid->GetHomeDirectory();
2187 workdir += fGridWorkingDir;
0d5d317c 2188 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2189 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2190 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2191 }
2192}