]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Added two missing includes to allow macro compilation (thanks to Laurent for remarkin...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
59 fArguments(),
60 fAnalysisMacro(),
61 fAnalysisSource(),
62 fAdditionalLibs(),
63 fSplitMode(),
64 fAPIVersion(),
65 fROOTVersion(),
66 fAliROOTVersion(),
67 fUser(),
68 fGridWorkingDir(),
69 fGridDataDir(),
70 fDataPattern(),
71 fGridOutputDir(),
72 fOutputArchive(),
73 fOutputFiles(),
74 fInputFormat(),
e7c71df0 75 fDatasetName(),
c57f56b7 76 fJDLName(),
bb885a9e 77 fMergeExcludes(),
f965131e 78 fIncludePath(),
bb885a9e 79 fCloseSE(),
0df6ccf2 80 fFriendChainName(),
c6cb3634 81 fJobTag(),
4e5c5506 82 fInputFiles(0),
83 fPackages(0)
c57f56b7 84{
85// Dummy ctor.
86 SetDefaults();
87}
88
89//______________________________________________________________________________
90AliAnalysisAlien::AliAnalysisAlien(const char *name)
91 :AliAnalysisGrid(name),
92 fGridJDL(NULL),
93 fPrice(0),
94 fTTL(0),
95 fSplitMaxInputFileNumber(0),
96 fMaxInitFailed(0),
97 fMasterResubmitThreshold(0),
bb885a9e 98 fNtestFiles(0),
319593fb 99 fNrunsPerMaster(0),
16a4353c 100 fMaxMergeFiles(0),
d2a409b2 101 fNsubmitted(0),
a3e84053 102 fProductionMode(0),
c57f56b7 103 fRunNumbers(),
104 fExecutable(),
105 fArguments(),
106 fAnalysisMacro(),
107 fAnalysisSource(),
108 fAdditionalLibs(),
109 fSplitMode(),
110 fAPIVersion(),
111 fROOTVersion(),
112 fAliROOTVersion(),
113 fUser(),
114 fGridWorkingDir(),
115 fGridDataDir(),
116 fDataPattern(),
117 fGridOutputDir(),
118 fOutputArchive(),
119 fOutputFiles(),
120 fInputFormat(),
e7c71df0 121 fDatasetName(),
c57f56b7 122 fJDLName(),
bb885a9e 123 fMergeExcludes(),
f965131e 124 fIncludePath(),
bb885a9e 125 fCloseSE(),
0df6ccf2 126 fFriendChainName(),
c6cb3634 127 fJobTag(),
4e5c5506 128 fInputFiles(0),
129 fPackages(0)
c57f56b7 130{
131// Default ctor.
132 SetDefaults();
133}
134
135//______________________________________________________________________________
136AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
137 :AliAnalysisGrid(other),
138 fGridJDL(NULL),
139 fPrice(other.fPrice),
140 fTTL(other.fTTL),
141 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
142 fMaxInitFailed(other.fMaxInitFailed),
143 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 144 fNtestFiles(other.fNtestFiles),
319593fb 145 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 146 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 147 fNsubmitted(other.fNsubmitted),
a3e84053 148 fProductionMode(other.fProductionMode),
c57f56b7 149 fRunNumbers(other.fRunNumbers),
150 fExecutable(other.fExecutable),
151 fArguments(other.fArguments),
152 fAnalysisMacro(other.fAnalysisMacro),
153 fAnalysisSource(other.fAnalysisSource),
154 fAdditionalLibs(other.fAdditionalLibs),
155 fSplitMode(other.fSplitMode),
156 fAPIVersion(other.fAPIVersion),
157 fROOTVersion(other.fROOTVersion),
158 fAliROOTVersion(other.fAliROOTVersion),
159 fUser(other.fUser),
160 fGridWorkingDir(other.fGridWorkingDir),
161 fGridDataDir(other.fGridDataDir),
162 fDataPattern(other.fDataPattern),
163 fGridOutputDir(other.fGridOutputDir),
164 fOutputArchive(other.fOutputArchive),
165 fOutputFiles(other.fOutputFiles),
166 fInputFormat(other.fInputFormat),
e7c71df0 167 fDatasetName(other.fDatasetName),
c57f56b7 168 fJDLName(other.fJDLName),
bb885a9e 169 fMergeExcludes(other.fMergeExcludes),
f965131e 170 fIncludePath(other.fIncludePath),
bb885a9e 171 fCloseSE(other.fCloseSE),
0df6ccf2 172 fFriendChainName(other.fFriendChainName),
c6cb3634 173 fJobTag(other.fJobTag),
4e5c5506 174 fInputFiles(0),
175 fPackages(0)
c57f56b7 176{
177// Copy ctor.
178 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 179 fRunRange[0] = other.fRunRange[0];
180 fRunRange[1] = other.fRunRange[1];
c57f56b7 181 if (other.fInputFiles) {
182 fInputFiles = new TObjArray();
183 TIter next(other.fInputFiles);
184 TObject *obj;
185 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
186 fInputFiles->SetOwner();
187 }
4e5c5506 188 if (other.fPackages) {
189 fPackages = new TObjArray();
190 TIter next(other.fPackages);
191 TObject *obj;
192 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
193 fPackages->SetOwner();
194 }
c57f56b7 195}
196
197//______________________________________________________________________________
198AliAnalysisAlien::~AliAnalysisAlien()
199{
200// Destructor.
201 if (fGridJDL) delete fGridJDL;
202 if (fInputFiles) delete fInputFiles;
4e5c5506 203 if (fPackages) delete fPackages;
c57f56b7 204}
205
206//______________________________________________________________________________
207AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
208{
209// Assignment.
210 if (this != &other) {
211 AliAnalysisGrid::operator=(other);
212 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
213 fPrice = other.fPrice;
214 fTTL = other.fTTL;
215 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
216 fMaxInitFailed = other.fMaxInitFailed;
217 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 218 fNtestFiles = other.fNtestFiles;
a3e84053 219 fNrunsPerMaster = other.fNrunsPerMaster;
220 fMaxMergeFiles = other.fMaxMergeFiles;
221 fNsubmitted = other.fNsubmitted;
222 fProductionMode = other.fProductionMode;
c57f56b7 223 fRunNumbers = other.fRunNumbers;
224 fExecutable = other.fExecutable;
225 fArguments = other.fArguments;
226 fAnalysisMacro = other.fAnalysisMacro;
227 fAnalysisSource = other.fAnalysisSource;
228 fAdditionalLibs = other.fAdditionalLibs;
229 fSplitMode = other.fSplitMode;
230 fAPIVersion = other.fAPIVersion;
231 fROOTVersion = other.fROOTVersion;
232 fAliROOTVersion = other.fAliROOTVersion;
233 fUser = other.fUser;
234 fGridWorkingDir = other.fGridWorkingDir;
235 fGridDataDir = other.fGridDataDir;
236 fDataPattern = other.fDataPattern;
237 fGridOutputDir = other.fGridOutputDir;
238 fOutputArchive = other.fOutputArchive;
239 fOutputFiles = other.fOutputFiles;
240 fInputFormat = other.fInputFormat;
e7c71df0 241 fDatasetName = other.fDatasetName;
c57f56b7 242 fJDLName = other.fJDLName;
bb885a9e 243 fMergeExcludes = other.fMergeExcludes;
f965131e 244 fIncludePath = other.fIncludePath;
bb885a9e 245 fCloseSE = other.fCloseSE;
0df6ccf2 246 fFriendChainName = other.fFriendChainName;
c6cb3634 247 fJobTag = other.fJobTag;
c57f56b7 248 if (other.fInputFiles) {
249 fInputFiles = new TObjArray();
250 TIter next(other.fInputFiles);
251 TObject *obj;
252 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
253 fInputFiles->SetOwner();
254 }
4e5c5506 255 if (other.fPackages) {
256 fPackages = new TObjArray();
257 TIter next(other.fPackages);
258 TObject *obj;
259 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
260 fPackages->SetOwner();
261 }
c57f56b7 262 }
263 return *this;
264}
265
f965131e 266//______________________________________________________________________________
267void AliAnalysisAlien::AddIncludePath(const char *path)
268{
269// Add include path in the remote analysis macro.
270 TString p(path);
271 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
272 else fIncludePath += Form("-I%s ", path);
273}
274
c57f56b7 275//______________________________________________________________________________
276void AliAnalysisAlien::AddRunNumber(Int_t run)
277{
278// Add a run number to the list of runs to be processed.
279 if (fRunNumbers.Length()) fRunNumbers += " ";
280 fRunNumbers += Form("%d", run);
281}
282
283//______________________________________________________________________________
284void AliAnalysisAlien::AddDataFile(const char *lfn)
285{
286// Adds a data file to the input to be analysed. The file should be a valid LFN
287// or point to an existing file in the alien workdir.
288 if (!fInputFiles) fInputFiles = new TObjArray();
289 fInputFiles->Add(new TObjString(lfn));
290}
291
292//______________________________________________________________________________
293Bool_t AliAnalysisAlien::Connect()
294{
295// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
296 if (gGrid && gGrid->IsConnected()) return kTRUE;
297 if (!gSystem->Getenv("alien_API_USER")) {
298 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
299 gSystem->Getenv("UID"));
300 return kFALSE;
301 }
302 if (!gGrid) {
303 Info("Connect", "Trying to connect to AliEn ...");
304 TGrid::Connect("alien://");
305 }
306 if (!gGrid || !gGrid->IsConnected()) {
307 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
308 return kFALSE;
309 }
310 fUser = gGrid->GetUser();
311 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
312 return kTRUE;
313}
314
315//______________________________________________________________________________
316void AliAnalysisAlien::CdWork()
317{
318// Check validity of alien workspace. Create directory if possible.
319 if (!Connect()) {
320 Error("CdWork", "Alien connection required");
321 return;
322 }
323 TString homedir = gGrid->GetHomeDirectory();
324 TString workdir = homedir + fGridWorkingDir;
923e2ca5 325 if (DirectoryExists(workdir)) {
326 gGrid->Cd(workdir);
327 return;
328 }
329 // Work directory not existing - create it
330 gGrid->Cd(homedir);
331 if (gGrid->Mkdir(workdir)) {
332 gGrid->Cd(fGridWorkingDir);
333 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
334 } else {
335 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
336 workdir.Data(), homedir.Data());
337 fGridWorkingDir = "";
338 }
c57f56b7 339}
340
341//______________________________________________________________________________
342Bool_t AliAnalysisAlien::CheckInputData()
343{
344// Check validity of input data. If necessary, create xml files.
a8739e8a 345 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 346 if (!fGridDataDir.Length()) {
347 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
348 return kFALSE;
349 }
350 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
351 return kTRUE;
c57f56b7 352 }
353 // Process declared files
354 Bool_t is_collection = kFALSE;
355 Bool_t is_xml = kFALSE;
356 Bool_t use_tags = kFALSE;
357 Bool_t checked = kFALSE;
358 CdWork();
359 TString file;
360 TString workdir = gGrid->GetHomeDirectory();
361 workdir += fGridWorkingDir;
362 if (fInputFiles) {
363 TObjString *objstr;
364 TIter next(fInputFiles);
365 while ((objstr=(TObjString*)next())) {
366 file = workdir;
367 file += "/";
368 file += objstr->GetString();
369 // Store full lfn path
370 if (FileExists(file)) objstr->SetString(file);
371 else {
372 file = objstr->GetName();
373 if (!FileExists(objstr->GetName())) {
374 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
375 objstr->GetName(), workdir.Data());
376 return kFALSE;
377 }
378 }
379 Bool_t iscoll, isxml, usetags;
380 CheckDataType(file, iscoll, isxml, usetags);
381 if (!checked) {
382 checked = kTRUE;
383 is_collection = iscoll;
384 is_xml = isxml;
385 use_tags = usetags;
386 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
387 } else {
388 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
389 Error("CheckInputData", "Some conflict was found in the types of inputs");
390 return kFALSE;
391 }
392 }
393 }
394 }
395 // Process requested run numbers
a8739e8a 396 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 397 // Check validity of alien data directory
398 if (!fGridDataDir.Length()) {
399 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
400 return kFALSE;
401 }
923e2ca5 402 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 403 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
404 return kFALSE;
405 }
406 if (is_collection) {
407 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
408 return kFALSE;
409 }
410
411 if (checked && !is_xml) {
412 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
413 return kFALSE;
414 }
415 // Check validity of run number(s)
416 TObjArray *arr;
417 TObjString *os;
319593fb 418 Int_t nruns = 0;
419 TString schunk;
c57f56b7 420 TString path;
421 if (!checked) {
422 checked = kTRUE;
423 use_tags = fDataPattern.Contains("tag");
424 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
425 }
426 if (use_tags != fDataPattern.Contains("tag")) {
427 Error("CheckInputData", "Cannot mix input files using/not using tags");
428 return kFALSE;
429 }
430 if (fRunNumbers.Length()) {
a8739e8a 431 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 432 arr = fRunNumbers.Tokenize(" ");
433 TIter next(arr);
434 while ((os=(TObjString*)next())) {
435 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 436 if (!DirectoryExists(path)) {
437 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 438 continue;
c57f56b7 439 }
440 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
441 TString msg = "\n##### file: ";
442 msg += path;
443 msg += " type: xml_collection;";
444 if (use_tags) msg += " using_tags: Yes";
445 else msg += " using_tags: No";
446 Info("CheckDataType", msg.Data());
319593fb 447 if (fNrunsPerMaster<2) {
d2a409b2 448 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 449 } else {
450 nruns++;
451 if (((nruns-1)%fNrunsPerMaster) == 0) {
452 schunk = os->GetString();
453 }
454 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
455 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 456 AddDataFile(schunk);
319593fb 457 }
c57f56b7 458 }
459 delete arr;
a8739e8a 460 } else {
461 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
462 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
463 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 464 if (!DirectoryExists(path)) {
465// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 466 continue;
467 }
468 path = Form("%s/%d.xml", workdir.Data(),irun);
469 TString msg = "\n##### file: ";
470 msg += path;
471 msg += " type: xml_collection;";
472 if (use_tags) msg += " using_tags: Yes";
473 else msg += " using_tags: No";
474 Info("CheckDataType", msg.Data());
319593fb 475 if (fNrunsPerMaster<2) {
d2a409b2 476 AddDataFile(Form("%d.xml",irun));
319593fb 477 } else {
478 nruns++;
479 if (((nruns-1)%fNrunsPerMaster) == 0) {
480 schunk = Form("%d", irun);
481 }
482 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
483 schunk += Form("_%d.xml", irun);
d2a409b2 484 AddDataFile(schunk);
319593fb 485 }
a8739e8a 486 }
c57f56b7 487 }
488 return kTRUE;
489}
490
491//______________________________________________________________________________
492Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
493{
494// Create dataset for the grid data directory + run number.
495 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
496 if (!Connect()) {
497 Error("CreateDataset", "Cannot create dataset with no grid connection");
498 return kFALSE;
499 }
500
501 // Cd workspace
502 CdWork();
503 TString workdir = gGrid->GetHomeDirectory();
504 workdir += fGridWorkingDir;
505
506 // Compose the 'find' command arguments
507 TString command;
508 TString options = "-x collection ";
bb885a9e 509 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 510 TString conditions = "";
511
512 TString file;
513 TString path;
319593fb 514 Int_t nruns = 0;
515 TString schunk;
ab254fd1 516 TGridCollection *cbase=0, *cadd=0;
d2a409b2 517 if (!fRunNumbers.Length() && !fRunRange[0]) {
518 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
519 // Make a single data collection from data directory.
520 path = fGridDataDir;
923e2ca5 521 if (!DirectoryExists(path)) {
d2a409b2 522 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
523 return kFALSE;
524 }
923e2ca5 525// CdWork();
d2a409b2 526 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
527 else file = Form("%s.xml", gSystem->BaseName(path));
528 if (gSystem->AccessPathName(file)) {
529 command = "find ";
530 command += options;
531 command += path;
532 command += " ";
533 command += pattern;
534 command += conditions;
923e2ca5 535 Printf("command: %s", command.Data());
d2a409b2 536 TGridResult *res = gGrid->Command(command);
537 if (res) delete res;
538 // Write standard output to file
539 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
540 }
541 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
542 // Copy xml file to alien space
543 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
544 if (!FileExists(file)) {
545 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
546 return kFALSE;
547 }
548 // Update list of files to be processed.
549 }
550 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
551 return kTRUE;
552 }
c57f56b7 553 // Several runs
a8739e8a 554 if (fRunNumbers.Length()) {
555 TObjArray *arr = fRunNumbers.Tokenize(" ");
556 TObjString *os;
557 TIter next(arr);
558 while ((os=(TObjString*)next())) {
559 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 560 if (!DirectoryExists(path)) continue;
561// CdWork();
a8739e8a 562 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
563 else file = Form("%s.xml", os->GetString().Data());
319593fb 564 // If local collection file does not exist, create it via 'find' command.
565 if (gSystem->AccessPathName(file)) {
566 command = "find ";
567 command += options;
568 command += path;
569 command += pattern;
570 command += conditions;
571 TGridResult *res = gGrid->Command(command);
572 if (res) delete res;
573 // Write standard output to file
574 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
575 }
a8739e8a 576 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 577 // Check if there is one run per master job.
578 if (fNrunsPerMaster<2) {
579 if (FileExists(file)) {
580 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
581 continue;
582 }
583 // Copy xml file to alien space
584 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
585 if (!FileExists(file)) {
586 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
587 delete arr;
588 return kFALSE;
589 }
590 } else {
591 nruns++;
592 if (((nruns-1)%fNrunsPerMaster) == 0) {
593 schunk = os->GetString();
594 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
595 } else {
596 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
923e2ca5 597 Printf(" Merging collection <%s> into masterjob input...", file.Data());
319593fb 598 cbase->Add(cadd);
599 delete cadd;
600 }
601 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
602 continue;
603 }
604 schunk += Form("_%s.xml", os->GetString().Data());
605 if (FileExists(schunk)) {
606 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
607 continue;
608 }
923e2ca5 609 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 610 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
611 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
612 if (!FileExists(schunk)) {
613 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
614 delete arr;
615 return kFALSE;
616 }
617 }
a8739e8a 618 }
619 delete arr;
620 } else {
621 // Process a full run range.
622 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
623 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 624 if (!DirectoryExists(path)) continue;
625// CdWork();
a8739e8a 626 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
627 else file = Form("%d.xml", irun);
319593fb 628 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 629 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
630// gGrid->Rm(file);
631 continue;
632 }
319593fb 633 // If local collection file does not exist, create it via 'find' command.
634 if (gSystem->AccessPathName(file)) {
635 command = "find ";
636 command += options;
637 command += path;
638 command += pattern;
639 command += conditions;
640 TGridResult *res = gGrid->Command(command);
641 if (res) delete res;
642 // Write standard output to file
643 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
644 }
a8739e8a 645 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 646 // Check if there is one run per master job.
647 if (fNrunsPerMaster<2) {
648 if (FileExists(file)) {
649 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
650 continue;
651 }
652 // Copy xml file to alien space
653 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
654 if (!FileExists(file)) {
655 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
656 return kFALSE;
657 }
658 } else {
659 nruns++;
95e5b448 660 // Check if the collection for the chunk exist locally.
661 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
662 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
923e2ca5 663 Printf(" Merging collection <%s> into %d runs chunk...",file.Data(),fNrunsPerMaster);
319593fb 664 if (((nruns-1)%fNrunsPerMaster) == 0) {
665 schunk = Form("%d", irun);
666 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
667 } else {
668 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
669 cbase->Add(cadd);
670 delete cadd;
671 }
672 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
673 continue;
674 }
675 schunk += Form("_%d.xml", irun);
676 if (FileExists(schunk)) {
677 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
678 continue;
679 }
923e2ca5 680 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 681 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 682 if (FileExists(schunk)) {
683 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
684 continue;
685 }
319593fb 686 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
687 if (!FileExists(schunk)) {
688 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
689 return kFALSE;
690 }
691 }
c57f56b7 692 }
a8739e8a 693 }
c57f56b7 694 return kTRUE;
695}
696
697//______________________________________________________________________________
698Bool_t AliAnalysisAlien::CreateJDL()
699{
700// Generate a JDL file according to current settings. The name of the file is
701// specified by fJDLName.
702 Bool_t error = kFALSE;
703 TObjArray *arr = 0;
704 Bool_t copy = kTRUE;
705 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
706 Bool_t generate = kTRUE;
707 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
708 if (!Connect()) {
709 Error("CreateJDL", "Alien connection required");
710 return kFALSE;
711 }
712 // Check validity of alien workspace
713 CdWork();
714 TString workdir = gGrid->GetHomeDirectory();
715 workdir += fGridWorkingDir;
716 if (generate) {
717 TObjString *os;
718 if (!fInputFiles) {
719 Error("CreateJDL()", "Define some input files for your analysis.");
720 error = kTRUE;
721 }
722 // Compose list of input files
723 // Check if output files were defined
724 if (!fOutputFiles.Length()) {
725 Error("CreateJDL", "You must define at least one output file");
726 error = kTRUE;
727 }
728 // Check if an output directory was defined and valid
729 if (!fGridOutputDir.Length()) {
730 Error("CreateJDL", "You must define AliEn output directory");
731 error = kTRUE;
732 } else {
d2a409b2 733 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 734 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 735 if (gGrid->Mkdir(fGridOutputDir)) {
736 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
737 } else {
738 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 739 // error = kTRUE;
c57f56b7 740 }
741 }
742 gGrid->Cd(workdir);
743 }
744 // Exit if any error up to now
745 if (error) return kFALSE;
746 // Set JDL fields
747 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 748 fGridJDL->SetExecutable(fExecutable);
c57f56b7 749// fGridJDL->SetTTL((UInt_t)fTTL);
750 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
751 if (fMaxInitFailed > 0)
752 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
753 if (fSplitMaxInputFileNumber > 0)
754 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
755 if (fSplitMode.Length())
756 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
757// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
758 if (fAliROOTVersion.Length())
759 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
760 if (fROOTVersion.Length())
761 fGridJDL->AddToPackages("ROOT", fROOTVersion);
762 if (fAPIVersion.Length())
763 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
764 fGridJDL->SetInputDataListFormat(fInputFormat);
765 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 766 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 767 TString analysisFile = fExecutable;
768 analysisFile.ReplaceAll(".sh", ".root");
769 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 770 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
771 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
772 if (fAdditionalLibs.Length()) {
773 arr = fAdditionalLibs.Tokenize(" ");
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 if (os->GetString().Contains(".so")) continue;
777 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
778 }
779 delete arr;
780 }
4e5c5506 781 if (fPackages) {
782 TIter next(fPackages);
783 TObject *obj;
784 while ((obj=next()))
785 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
786 }
c57f56b7 787 if (fOutputArchive.Length()) {
788 arr = fOutputArchive.Tokenize(" ");
789 TIter next(arr);
790 while ((os=(TObjString*)next()))
43da816a 791 if (!os->GetString().Contains("@") && fCloseSE.Length())
792 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
793 else
794 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 795 delete arr;
796 }
c57f56b7 797 arr = fOutputFiles.Tokenize(" ");
798 TIter next(arr);
43da816a 799 while ((os=(TObjString*)next())) {
e1eaf596 800 // Ignore ouputs in jdl that are also in outputarchive
801 TString sout = os->GetString();
802 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
803 if (fOutputArchive.Contains(sout)) continue;
43da816a 804 if (!os->GetString().Contains("@") && fCloseSE.Length())
805 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
806 else
807 fGridJDL->AddToOutputSandbox(os->GetString());
808 }
c57f56b7 809 delete arr;
810// fGridJDL->SetPrice((UInt_t)fPrice);
811 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 812 TString validationScript = fExecutable;
813 validationScript.ReplaceAll(".sh", "_validation.sh");
814 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 815 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 816 // Write a jdl with 2 input parameters: collection name and output dir name.
817 WriteJDL(copy);
c57f56b7 818 }
819 // Copy jdl to grid workspace
a8739e8a 820 if (copy) {
c57f56b7 821 if (fAdditionalLibs.Length()) {
e7c71df0 822 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 823 TObjString *os;
824 TIter next(arr);
825 while ((os=(TObjString*)next())) {
c57f56b7 826 if (os->GetString().Contains(".so")) continue;
4e5c5506 827 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 828 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
829 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
830 }
831 delete arr;
832 }
4e5c5506 833 if (fPackages) {
834 TIter next(fPackages);
835 TObject *obj;
836 while ((obj=next())) {
837 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
838 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
839 }
840 }
c57f56b7 841 }
842 return kTRUE;
843}
844
a8739e8a 845//______________________________________________________________________________
d2a409b2 846Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 847{
848// Writes one or more JDL's corresponding to findex. If findex is negative,
849// all run numbers are considered in one go (jdl). For non-negative indices
850// they correspond to the indices in the array fInputFiles.
851 if (!fInputFiles) return kFALSE;
852 TObjString *os;
a8739e8a 853 TString workdir = gGrid->GetHomeDirectory();
854 workdir += fGridWorkingDir;
d2a409b2 855
856 if (!fRunNumbers.Length() && !fRunRange[0]) {
857 // One jdl with no parameters in case input data is specified by name.
a8739e8a 858 TIter next(fInputFiles);
859 while ((os=(TObjString*)next()))
860 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 861 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 862 } else {
d2a409b2 863 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
864 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 865 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 866 }
867
868
869 // Generate the JDL as a string
870 TString sjdl = fGridJDL->Generate();
871 Int_t index;
872 index = sjdl.Index("Executable");
873 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
874 index = sjdl.Index("Split ");
875 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
876 index = sjdl.Index("SplitMaxInputFileNumber");
877 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
878 index = sjdl.Index("InputDataCollection");
879 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 880 index = sjdl.Index("InputFile");
881 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
882 index = sjdl.Index("InputDataList ");
883 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
884 index = sjdl.Index("InputDataListFormat");
885 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
886 index = sjdl.Index("Price");
887 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
888 index = sjdl.Index("Requirements");
889 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
890 index = sjdl.Index("Packages");
891 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
892 index = sjdl.Index("User =");
893 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
894 index = sjdl.Index("TTL");
895 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
896 index = sjdl.Index("OutputFile");
897 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
898 index = sjdl.Index("OutputDir");
899 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
900 index = sjdl.Index("OutputArchive");
901 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
902 index = sjdl.Index("MaxInitFailed");
903 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
904 index = sjdl.Index("MasterResubmitThreshold");
905 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
906 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
907 index = sjdl.Index("Validationcommand");
908 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
909 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
910 sjdl.ReplaceAll("(member", "\n (member");
911 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
912 sjdl.ReplaceAll("{", "{\n ");
913 sjdl.ReplaceAll("};", "\n};");
914 sjdl.ReplaceAll("{\n \n", "{\n");
915 sjdl.ReplaceAll("\n\n", "\n");
916 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
917 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 918 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 919 index = sjdl.Index("JDLVariables");
920 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
921 // Write jdl to file
a8739e8a 922 ofstream out;
d2a409b2 923 out.open(fJDLName.Data(), ios::out);
a8739e8a 924 if (out.bad()) {
d2a409b2 925 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 926 return kFALSE;
927 }
928 out << sjdl << endl;
929
930 // Copy jdl to grid workspace
931 if (!copy) {
932 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
933 } else {
d2a409b2 934 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
935 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 936 if (fProductionMode)
b5fe9cba 937 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 938 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 939 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 940 }
941 return kTRUE;
942}
943
c57f56b7 944//______________________________________________________________________________
5513444a 945Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 946{
947// Returns true if file exists.
5513444a 948 if (!gGrid) return kFALSE;
c57f56b7 949 TGridResult *res = gGrid->Ls(lfn);
950 if (!res) return kFALSE;
951 TMap *map = dynamic_cast<TMap*>(res->At(0));
952 if (!map) {
953 delete res;
954 return kFALSE;
955 }
956 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
957 if (!objs || !objs->GetString().Length()) {
958 delete res;
959 return kFALSE;
960 }
961 delete res;
962 return kTRUE;
963}
964
923e2ca5 965//______________________________________________________________________________
966Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
967{
968// Returns true if directory exists. Can be also a path.
969 if (!gGrid) return kFALSE;
970 // Check if dirname is a path
971 TString dirstripped = dirname;
972 dirstripped = dirstripped.Strip();
973 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
974 TString dir = gSystem->BaseName(dirstripped);
975 dir += "/";
976 TString path = gSystem->DirName(dirstripped);
977 TGridResult *res = gGrid->Ls(path, "-F");
978 if (!res) return kFALSE;
979 TIter next(res);
980 TMap *map;
981 TObject *obj;
982 while ((map=dynamic_cast<TMap*>(next()))) {
983 obj = map->GetValue("name");
984 if (!obj) break;
985 if (dir == obj->GetName()) {
986 delete res;
987 return kTRUE;
988 }
989 }
990 delete res;
991 return kFALSE;
992}
993
c57f56b7 994//______________________________________________________________________________
995void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
996{
997// Check input data type.
998 is_collection = kFALSE;
999 is_xml = kFALSE;
1000 use_tags = kFALSE;
1001 if (!gGrid) {
1002 Error("CheckDataType", "No connection to grid");
1003 return;
1004 }
1005 is_collection = IsCollection(lfn);
1006 TString msg = "\n##### file: ";
1007 msg += lfn;
1008 if (is_collection) {
1009 msg += " type: raw_collection;";
1010 // special treatment for collections
1011 is_xml = kFALSE;
1012 // check for tag files in the collection
1013 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1014 if (!res) {
1015 msg += " using_tags: No (unknown)";
1016 Info("CheckDataType", msg.Data());
1017 return;
1018 }
1019 const char* typeStr = res->GetKey(0, "origLFN");
1020 if (!typeStr || !strlen(typeStr)) {
1021 msg += " using_tags: No (unknown)";
1022 Info("CheckDataType", msg.Data());
1023 return;
1024 }
1025 TString file = typeStr;
1026 use_tags = file.Contains(".tag");
1027 if (use_tags) msg += " using_tags: Yes";
1028 else msg += " using_tags: No";
1029 Info("CheckDataType", msg.Data());
1030 return;
1031 }
1032 TString slfn(lfn);
1033 slfn.ToLower();
1034 is_xml = slfn.Contains(".xml");
1035 if (is_xml) {
1036 // Open xml collection and check if there are tag files inside
1037 msg += " type: xml_collection;";
1038 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1039 if (!coll) {
1040 msg += " using_tags: No (unknown)";
1041 Info("CheckDataType", msg.Data());
1042 return;
1043 }
1044 TMap *map = coll->Next();
1045 if (!map) {
1046 msg += " using_tags: No (unknown)";
1047 Info("CheckDataType", msg.Data());
1048 return;
1049 }
1050 map = (TMap*)map->GetValue("");
1051 TString file;
1052 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1053 use_tags = file.Contains(".tag");
1054 delete coll;
1055 if (use_tags) msg += " using_tags: Yes";
1056 else msg += " using_tags: No";
1057 Info("CheckDataType", msg.Data());
1058 return;
1059 }
1060 use_tags = slfn.Contains(".tag");
1061 if (slfn.Contains(".root")) msg += " type: root file;";
1062 else msg += " type: unhnown file;";
1063 if (use_tags) msg += " using_tags: Yes";
1064 else msg += " using_tags: No";
1065 Info("CheckDataType", msg.Data());
1066}
1067
4e5c5506 1068//______________________________________________________________________________
1069void AliAnalysisAlien::EnablePackage(const char *package)
1070{
1071// Enables a par file supposed to exist in the current directory.
1072 TString pkg(package);
1073 pkg.ReplaceAll(".par", "");
1074 pkg += ".par";
1075 if (gSystem->AccessPathName(pkg)) {
1076 Error("EnablePackage", "Package %s not found", pkg.Data());
1077 return;
1078 }
1079 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1080 Info("EnablePackage", "AliEn plugin will use .par packages");
1081 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1082 if (!fPackages) {
1083 fPackages = new TObjArray();
1084 fPackages->SetOwner();
1085 }
1086 fPackages->Add(new TObjString(pkg));
1087}
1088
d2a409b2 1089//______________________________________________________________________________
1090const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1091{
1092// Get job status for all jobs with jobid>jobidstart.
1093 static char mstatus[20];
1094 mstatus[0] = '\0';
1095 nrunning = 0;
1096 nwaiting = 0;
1097 nerror = 0;
1098 ndone = 0;
1099 TGridJobStatusList *list = gGrid->Ps("");
1100 if (!list) return mstatus;
1101 Int_t nentries = list->GetSize();
1102 TGridJobStatus *status;
1103 Int_t pid;
1104 for (Int_t ijob=0; ijob<nentries; ijob++) {
1105 status = (TGridJobStatus *)list->At(ijob);
1106 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1107 if (pid<jobidstart) continue;
1108 if (pid == lastid) {
1109 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1110 }
1111 switch (status->GetStatus()) {
1112 case TGridJobStatus::kWAITING:
1113 nwaiting++; break;
1114 case TGridJobStatus::kRUNNING:
1115 nrunning++; break;
1116 case TGridJobStatus::kABORTED:
1117 case TGridJobStatus::kFAIL:
1118 case TGridJobStatus::kUNKNOWN:
1119 nerror++; break;
1120 case TGridJobStatus::kDONE:
1121 ndone++;
1122 }
1123 }
1124 list->Delete();
1125 delete list;
1126 return mstatus;
1127}
1128
c57f56b7 1129//______________________________________________________________________________
1130Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1131{
1132// Returns true if file is a collection. Functionality duplicated from
1133// TAlien::Type() because we don't want to directly depend on TAlien.
1134 if (!gGrid) {
1135 Error("IsCollection", "No connection to grid");
1136 return kFALSE;
1137 }
1138 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1139 if (!res) return kFALSE;
1140 const char* typeStr = res->GetKey(0, "type");
1141 if (!typeStr || !strlen(typeStr)) return kFALSE;
1142 if (!strcmp(typeStr, "collection")) return kTRUE;
1143 delete res;
1144 return kFALSE;
1145}
1146
16a4353c 1147//______________________________________________________________________________
1148void AliAnalysisAlien::Print(Option_t *) const
1149{
1150// Print current plugin settings.
923e2ca5 1151 Printf("### AliEn analysis plugin current settings ###");
1152 Printf("= Production mode:______________________________ %d", fProductionMode);
1153 Printf("= Version of API requested: ____________________ %s", fAPIVersion.Data());
1154 Printf("= Version of ROOT requested: ___________________ %s", fROOTVersion.Data());
1155 Printf("= Version of AliRoot requested: ________________ %s", fAliROOTVersion.Data());
16a4353c 1156 if (fUser.Length())
923e2ca5 1157 Printf("= User running the plugin: _____________________ %s", fUser.Data());
1158 Printf("= Grid workdir relative to user $HOME: _________ %s", fGridWorkingDir.Data());
1159 Printf("= Grid output directory relative to workdir: ___ %s", fGridOutputDir.Data());
1160 Printf("= Data base directory path requested: __________ %s", fGridDataDir.Data());
1161 Printf("= Data search pattern: _________________________ %s", fDataPattern.Data());
1162 Printf("= Input data format: ___________________________ %s", fInputFormat.Data());
16a4353c 1163 if (fRunNumbers.Length())
923e2ca5 1164 Printf("= Run numbers to be processed: _________________ %s", fRunNumbers.Data());
16a4353c 1165 if (fRunRange[0])
923e2ca5 1166 Printf("= Run range to be processed: ___________________ %d-%d", fRunRange[0], fRunRange[1]);
16a4353c 1167 if (!fRunRange[0] && !fRunNumbers.Length()) {
1168 TIter next(fInputFiles);
1169 TObject *obj;
1170 TString list;
1171 while ((obj=next())) list += obj->GetName();
923e2ca5 1172 Printf("= Input files to be processed: _________________ %s", list.Data());
16a4353c 1173 }
1174 if (TestBit(AliAnalysisGrid::kTest))
923e2ca5 1175 Printf("= Number of input files used in test mode: _____ %d", fNtestFiles);
1176 Printf("= List of output files to be registered: _______ %s", fOutputFiles.Data());
1177 Printf("= List of outputs going to be archived: ________ %s", fOutputArchive.Data());
1178 Printf("= List of outputs that should not be merged: ___ %s", fMergeExcludes.Data());
1179 Printf("=====================================================================");
1180 Printf("= Job price: ___________________________________ %d", fPrice);
1181 Printf("= Time to live (TTL): __________________________ %d", fTTL);
1182 Printf("= Max files per subjob: ________________________ %d", fSplitMaxInputFileNumber);
16a4353c 1183 if (fMaxInitFailed>0)
923e2ca5 1184 Printf("= Max number of subjob fails to kill: __________ %d", fMaxInitFailed);
16a4353c 1185 if (fMasterResubmitThreshold>0)
923e2ca5 1186 Printf("= Resubmit master job if failed subjobs >_______ %d", fMasterResubmitThreshold);
319593fb 1187 if (fNrunsPerMaster>0)
923e2ca5 1188 Printf("= Number of runs per master job: _______________ %d", fNrunsPerMaster);
1189 Printf("= Number of files in one chunk to be merged: ___ %d", fMaxMergeFiles);
1190 Printf("= Name of the generated execution script: ______ %s",fExecutable.Data());
16a4353c 1191 if (fArguments.Length())
923e2ca5 1192 Printf("= Arguments for the execution script: __________ %s",fArguments.Data());
1193 Printf("= Name of the generated analysis macro: ________ %s",fAnalysisMacro.Data());
1194 Printf("= User analysis files to be deployed: __________ %s",fAnalysisSource.Data());
1195 Printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>",fAdditionalLibs.Data());
1196 Printf("= Master jobs split mode: ______________________ %s",fSplitMode.Data());
16a4353c 1197 if (fDatasetName)
923e2ca5 1198 Printf("= Custom name for the dataset to be created: ___ %s", fDatasetName.Data());
1199 Printf("= Name of the generated JDL: ___________________ %s", fJDLName.Data());
16a4353c 1200 if (fIncludePath.Data())
923e2ca5 1201 Printf("= Include path for runtime task compilation: ___ %s", fIncludePath.Data());
16a4353c 1202 if (fCloseSE.Length())
923e2ca5 1203 Printf("= Force job outputs to storage element: ________ %s", fCloseSE.Data());
16a4353c 1204 if (fFriendChainName.Length())
923e2ca5 1205 Printf("= Open friend chain file on worker: ____________ %s", fFriendChainName.Data());
16a4353c 1206 if (fPackages) {
1207 TIter next(fPackages);
1208 TObject *obj;
1209 TString list;
1210 while ((obj=next())) list += obj->GetName();
923e2ca5 1211 Printf("= Par files to be used: ________________________ %s", list.Data());
16a4353c 1212 }
1213}
1214
c57f56b7 1215//______________________________________________________________________________
1216void AliAnalysisAlien::SetDefaults()
1217{
1218// Set default values for everything. What cannot be filled will be left empty.
1219 if (fGridJDL) delete fGridJDL;
1220 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1221 fPrice = 1;
1222 fTTL = 30000;
1223 fSplitMaxInputFileNumber = 100;
1224 fMaxInitFailed = 0;
1225 fMasterResubmitThreshold = 0;
bb885a9e 1226 fNtestFiles = 10;
a8739e8a 1227 fRunRange[0] = 0;
1228 fRunRange[1] = 0;
319593fb 1229 fNrunsPerMaster = 1;
16a4353c 1230 fMaxMergeFiles = 100;
c57f56b7 1231 fRunNumbers = "";
1232 fExecutable = "analysis.sh";
1233 fArguments = "";
1234 fAnalysisMacro = "myAnalysis.C";
1235 fAnalysisSource = "";
1236 fAdditionalLibs = "";
1237 fSplitMode = "se";
1238 fAPIVersion = "";
1239 fROOTVersion = "";
1240 fAliROOTVersion = "";
1241 fUser = ""; // Your alien user name
1242 fGridWorkingDir = "";
1243 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1244 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1245 fFriendChainName = "";
c57f56b7 1246 fGridOutputDir = "output";
1247 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1248 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1249 fInputFormat = "xml-single";
1250 fJDLName = "analysis.jdl";
c6cb3634 1251 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1252 fMergeExcludes = "";
c57f56b7 1253}
1254
1255//______________________________________________________________________________
1256Bool_t AliAnalysisAlien::MergeOutputs()
1257{
1258// Merge analysis outputs existing in the AliEn space.
1259 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1260 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1261 if (!Connect()) {
1262 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1263 return kFALSE;
1264 }
1265 // Get the output path
d2a409b2 1266 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1267 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1268 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1269 return kFALSE;
1270 }
1271 if (!fOutputFiles.Length()) {
1272 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1273 return kFALSE;
1274 }
1275 TObjArray *list = fOutputFiles.Tokenize(" ");
1276 TIter next(list);
1277 TObjString *str;
1278 TString command;
1279 TString output_file;
16a4353c 1280 TString output_chunk;
1281 TString previous_chunk;
1282 Int_t count_chunk = 0;
1283 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1284 Bool_t merged = kTRUE;
1285 while((str=(TObjString*)next())) {
1286 output_file = str->GetString();
1287 Int_t index = output_file.Index("@");
1288 if (index > 0) output_file.Remove(index);
319593fb 1289 // Skip already merged outputs
1290 if (!gSystem->AccessPathName(output_file)) {
1291 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1292 continue;
1293 }
bb885a9e 1294 if (fMergeExcludes.Length() &&
1295 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1296 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1297 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
923e2ca5 1298 Printf("command: %s", command.Data());
c57f56b7 1299 TGridResult *res = gGrid->Command(command);
1300 if (!res) continue;
1301 TFileMerger *fm = 0;
1302 TIter nextmap(res);
ab254fd1 1303 TMap *map = 0;
16a4353c 1304 previous_chunk = "";
1305 count_chunk = 0;
319593fb 1306 // Check if there is a merge operation to resume
1307 output_chunk = output_file;
1308 output_chunk.ReplaceAll(".root", "_*.root");
1309 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1310 while (1) {
1311 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1312 if (!map) {
1313 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1314 delete res;
1315 return kFALSE;
1316 }
1317 output_chunk = output_file;
1318 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
923e2ca5 1319 Printf("%s", output_chunk.Data());
319593fb 1320 count_chunk++;
1321 if (gSystem->AccessPathName(output_chunk)) continue;
1322 // Merged file with chunks up to <count_chunk> found
923e2ca5 1323 Printf("Resume merging of <%s> from <%s>", output_file.Data(), output_chunk.Data());
319593fb 1324 previous_chunk = output_chunk;
1325 break;
1326 }
1327 }
16a4353c 1328 count_zero = fMaxMergeFiles;
c57f56b7 1329 while ((map=(TMap*)nextmap())) {
16a4353c 1330 // Loop 'find' results and get next LFN
1331 if (count_zero == fMaxMergeFiles) {
1332 // First file in chunk - create file merger and add previous chunk if any.
1333 fm = new TFileMerger(kFALSE);
1334 fm->SetFastMethod(kTRUE);
1335 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1336 output_chunk = output_file;
1337 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1338 }
1339 // If last file found, put merged results in the output file
1340 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1341 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1342 if (!objs || !objs->GetString().Length()) {
16a4353c 1343 // Nothing found - skip this output
c57f56b7 1344 delete res;
16a4353c 1345 delete fm;
1346 break;
1347 }
1348 // Add file to be merged and decrement chunk counter.
1349 fm->AddFile(objs->GetString());
1350 count_zero--;
1351 if (count_zero==0 || map == res->Last()) {
1352 fm->OutputFile(output_chunk);
1353 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1354 // Nothing found - skip this output
1355 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1356 delete res;
1357 delete fm;
1358 break;
1359 }
1360 // Merge the outputs, then go to next chunk
1361 if (!fm->Merge()) {
1362 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1363 delete res;
1364 delete fm;
1365 merged = kFALSE;
1366 break;
1367 } else {
1368 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1369 gSystem->Unlink(previous_chunk);
16a4353c 1370 }
1371 if (map == res->Last()) {
1372 delete res;
1373 delete fm;
1374 break;
1375 }
1376 count_chunk++;
1377 count_zero = fMaxMergeFiles;
1378 previous_chunk = output_chunk;
c57f56b7 1379 }
c57f56b7 1380 }
c57f56b7 1381 }
1382 if (!merged) {
1383 Error("MergeOutputs", "Terminate() will NOT be executed");
1384 }
1385 return merged;
1386}
1387
bb885a9e 1388//______________________________________________________________________________
1389void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1390{
1391// Use the output files connected to output containers from the analysis manager
1392// rather than the files defined by SetOutputFiles
1393 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1394 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1395 analysis manager");
1396 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1397}
1398
5513444a 1399//______________________________________________________________________________
1400Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1401{
1402// Start remote grid analysis.
1403
43da816a 1404 // Check if output files have to be taken from the analysis manager
1405 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1406 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1407 if (!mgr || !mgr->IsInitialized()) {
1408 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1409 return kFALSE;
43da816a 1410 }
1411 fOutputFiles = "";
1412 TIter next(mgr->GetOutputs());
1413 AliAnalysisDataContainer *output;
1414 while ((output=(AliAnalysisDataContainer*)next())) {
1415 const char *filename = output->GetFileName();
1416 if (!(strcmp(filename, "default"))) {
1417 if (!mgr->GetOutputEventHandler()) continue;
1418 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1419 }
1420 if (fOutputFiles.Length()) fOutputFiles += " ";
1421 fOutputFiles += filename;
1422 }
c07b9ce2 1423 // Add extra files registered to the analysis manager
1424 if (mgr->GetExtraFiles().Length()) {
1425 if (fOutputFiles.Length()) fOutputFiles += " ";
1426 fOutputFiles += mgr->GetExtraFiles();
1427 }
43da816a 1428 }
f7b1cbc2 1429// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1430 if (TestBit(AliAnalysisGrid::kOffline)) {
1431 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1432 \n there nor any job run. You can revise the JDL and analysis \
1433 \n macro then run the same in \"submit\" mode.");
1434 } else if (TestBit(AliAnalysisGrid::kTest)) {
1435 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1436 \n dataset.");
1437 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1438 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1439 \n space and job submitted.");
1440 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1441 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1442 return kTRUE;
c57f56b7 1443 } else {
1444 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1445 }
1446
1447 if (!Connect()) {
1448 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1449 return kFALSE;
16a4353c 1450 }
1451 Print();
c57f56b7 1452 if (!CheckInputData()) {
1453 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1454 return kFALSE;
c57f56b7 1455 }
1456 CreateDataset(fDataPattern);
1457 WriteAnalysisFile();
1458 WriteAnalysisMacro();
1459 WriteExecutable();
1460 WriteValidationScript();
5513444a 1461 if (!CreateJDL()) return kFALSE;
1462 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1463 if (TestBit(AliAnalysisGrid::kTest)) {
1464 // Locally testing the analysis
1465 Info("StartAnalysis", "\n_______________________________________________________________________ \
1466 \n Running analysis script in a daughter shell as on a worker node \
1467 \n_______________________________________________________________________");
1468 TObjArray *list = fOutputFiles.Tokenize(" ");
1469 TIter next(list);
1470 TObjString *str;
1471 TString output_file;
1472 while((str=(TObjString*)next())) {
1473 output_file = str->GetString();
1474 Int_t index = output_file.Index("@");
1475 if (index > 0) output_file.Remove(index);
43da816a 1476 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1477 }
1478 delete list;
1479 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1480 TString validationScript = fExecutable;
1481 validationScript.ReplaceAll(".sh", "_validation.sh");
1482 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1483// gSystem->Exec("cat stdout");
5513444a 1484 return kFALSE;
c57f56b7 1485 }
5513444a 1486 // Check if submitting is managed by LPM manager
a3e84053 1487 if (fProductionMode) {
5513444a 1488 TString prodfile = fJDLName;
1489 prodfile.ReplaceAll(".jdl", ".prod");
1490 WriteProductionFile(prodfile);
1491 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1492 return kFALSE;
1493 }
a8739e8a 1494 // Submit AliEn job(s)
d2a409b2 1495 gGrid->Cd(fGridOutputDir);
a8739e8a 1496 TGridResult *res;
c57f56b7 1497 TString jobID = "";
d2a409b2 1498 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1499 // Submit a given xml or a set of runs
a8739e8a 1500 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
923e2ca5 1501 Printf("*************************** %s",Form("submit %s", fJDLName.Data()));
a8739e8a 1502 if (res) {
1503 const char *cjobId = res->GetKey(0,"jobId");
1504 if (!cjobId) {
1505 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1506 return kFALSE;
a8739e8a 1507 } else {
1508 Info("StartAnalysis", "\n_______________________________________________________________________ \
1509 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1510 \n_______________________________________________________________________",
1511 fJDLName.Data(), cjobId);
1512 jobID = cjobId;
1513 }
1514 delete res;
1515 }
1516 } else {
d2a409b2 1517 // Submit for a range of enumeration of runs.
1518 Submit();
c57f56b7 1519 }
a8739e8a 1520
c57f56b7 1521 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1522 \n You may exit at any time and terminate the job later using the option <terminate> \
1523 \n ##################################################################################", jobID.Data());
bb885a9e 1524 gSystem->Exec("aliensh");
5513444a 1525 return kTRUE;
c57f56b7 1526}
1527
d2a409b2 1528//______________________________________________________________________________
1529void AliAnalysisAlien::Submit()
1530{
1531// Submit all master jobs.
1532 Int_t nmasterjobs = fInputFiles->GetEntries();
1533 Long_t tshoot = gSystem->Now();
1534 if (!fNsubmitted) SubmitNext();
1535 while (fNsubmitted < nmasterjobs) {
1536 Long_t now = gSystem->Now();
1537 if ((now-tshoot)>30000) {
1538 tshoot = now;
1539 SubmitNext();
1540 }
1541 }
1542}
1543
1544//______________________________________________________________________________
1545void AliAnalysisAlien::SubmitNext()
1546{
1547// Submit next bunch of master jobs if the queue is free.
1548 static Bool_t iscalled = kFALSE;
1549 static Int_t firstmaster = 0;
1550 static Int_t lastmaster = 0;
1551 static Int_t npermaster = 0;
1552 if (iscalled) return;
1553 iscalled = kTRUE;
1554 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1555 Int_t ntosubmit = 0;
1556 TGridResult *res;
1557 TString jobID = "";
1558 if (!fNsubmitted) ntosubmit = 1;
1559 else {
1560 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
923e2ca5 1561 Printf("=== master %d: %s", lastmaster, status.Data());
d2a409b2 1562 // If last master not split, just return
1563 if (status != "SPLIT") {iscalled = kFALSE; return;}
1564 // No more than 100 waiting jobs
1565 if (nwaiting>100) {iscalled = kFALSE; return;}
1566 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1567 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
923e2ca5 1568 Printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs",
d2a409b2 1569 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1570 }
1571 Int_t nmasterjobs = fInputFiles->GetEntries();
1572 for (Int_t i=0; i<ntosubmit; i++) {
1573 // Submit for a range of enumeration of runs.
1574 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1575 TString query;
1576 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
923e2ca5 1577 Printf("********* %s",query.Data());
d2a409b2 1578 res = gGrid->Command(query);
1579 if (res) {
98ca124f 1580 TString cjobId1 = res->GetKey(0,"jobId");
1581 if (!cjobId1.Length()) {
d2a409b2 1582 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1583 iscalled = kFALSE;
1584 return;
1585 } else {
1586 Info("StartAnalysis", "\n_______________________________________________________________________ \
1587 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1588 \n_______________________________________________________________________",
98ca124f 1589 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1590 jobID += cjobId1;
1591 jobID += " ";
98ca124f 1592 lastmaster = cjobId1.Atoi();
d2a409b2 1593 if (!firstmaster) firstmaster = lastmaster;
1594 fNsubmitted++;
1595 }
1596 delete res;
1597 }
1598 }
1599 iscalled = kFALSE;
1600}
1601
c57f56b7 1602//______________________________________________________________________________
1603void AliAnalysisAlien::WriteAnalysisFile()
1604{
f10e8481 1605// Write current analysis manager into the file <analysisFile>
1606 TString analysisFile = fExecutable;
1607 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1608 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1609 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1610 if (!mgr || !mgr->IsInitialized()) {
1611 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1612 return;
1613 }
1614 // Check analysis type
1615 TObject *handler;
1616 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1617 handler = (TObject*)mgr->GetInputEventHandler();
1618 if (handler) {
1619 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1620 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1621 }
1622 TDirectory *cdir = gDirectory;
f10e8481 1623 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1624 if (file) {
1625 mgr->Write();
1626 delete file;
1627 }
1628 if (cdir) cdir->cd();
f10e8481 1629 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1630 }
1631 Bool_t copy = kTRUE;
1632 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1633 if (copy) {
1634 CdWork();
1635 TString workdir = gGrid->GetHomeDirectory();
1636 workdir += fGridWorkingDir;
f10e8481 1637 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1638 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1639 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1640 }
1641}
1642
1643//______________________________________________________________________________
1644void AliAnalysisAlien::WriteAnalysisMacro()
1645{
1646// Write the analysis macro that will steer the analysis in grid mode.
1647 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1648 ofstream out;
1649 out.open(fAnalysisMacro.Data(), ios::out);
1650 if (!out.good()) {
1651 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1652 return;
1653 }
1654 TString func = fAnalysisMacro;
1655 TString type = "ESD";
1656 TString comment = "// Analysis using ";
1657 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1658 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1659 type = "AOD";
1660 comment += "AOD";
1661 }
0df6ccf2 1662 if (type!="AOD" && fFriendChainName!="") {
1663 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1664 return;
1665 }
c57f56b7 1666 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1667 else comment += " data";
1668 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1669 func.ReplaceAll(".C", "");
1670 out << "void " << func.Data() << "()" << endl;
1671 out << "{" << endl;
1672 out << comment.Data() << endl;
1673 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1674 out << " TStopwatch timer;" << endl;
1675 out << " timer.Start();" << endl << endl;
c57f56b7 1676 out << "// load base root libraries" << endl;
1677 out << " gSystem->Load(\"libTree\");" << endl;
1678 out << " gSystem->Load(\"libGeom\");" << endl;
1679 out << " gSystem->Load(\"libVMC\");" << endl;
1680 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1681 out << "// Load analysis framework libraries" << endl;
4e5c5506 1682 if (!fPackages) {
4e5c5506 1683 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1684 out << " gSystem->Load(\"libESD\");" << endl;
1685 out << " gSystem->Load(\"libAOD\");" << endl;
1686 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1687 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1688 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1689 } else {
4e5c5506 1690 TIter next(fPackages);
1691 TObject *obj;
57377eb5 1692 TString pkgname;
1693 Bool_t hasSTEERBase = kFALSE;
1694 Bool_t hasESD = kFALSE;
1695 Bool_t hasAOD = kFALSE;
1696 Bool_t hasANALYSIS = kFALSE;
1697 Bool_t hasANALYSISalice = kFALSE;
1698 Bool_t hasCORRFW = kFALSE;
1699 while ((obj=next())) {
1700 pkgname = obj->GetName();
4478e6f1 1701 if (pkgname == "STEERBase" ||
1702 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1703 if (pkgname == "ESD" ||
1704 pkgname == "ESD.par") hasESD = kTRUE;
1705 if (pkgname == "AOD" ||
1706 pkgname == "AOD.par") hasAOD = kTRUE;
1707 if (pkgname == "ANALYSIS" ||
1708 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1709 if (pkgname == "ANALYSISalice" ||
1710 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1711 if (pkgname == "CORRFW" ||
1712 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1713 }
1714 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1715 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1716 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1717 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1718 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1719 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1720 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1721 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1722 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1723 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1724 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1725 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1726 out << "// Compile other par packages" << endl;
1727 next.Reset();
fcc9bb6f 1728 while ((obj=next())) {
1729 pkgname = obj->GetName();
4478e6f1 1730 if (pkgname == "STEERBase" ||
1731 pkgname == "STEERBase.par" ||
1732 pkgname == "ESD" ||
1733 pkgname == "ESD.par" ||
1734 pkgname == "AOD" ||
1735 pkgname == "AOD.par" ||
1736 pkgname == "ANALYSIS" ||
1737 pkgname == "ANALYSIS.par" ||
1738 pkgname == "ANALYSISalice" ||
1739 pkgname == "ANALYSISalice.par" ||
1740 pkgname == "CORRFW" ||
1741 pkgname == "CORRFW.par") continue;
4e5c5506 1742 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1743 }
4e5c5506 1744 }
43da816a 1745 out << "// include path" << endl;
1746 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1747 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1748 if (fAdditionalLibs.Length()) {
1749 out << "// Add aditional AliRoot libraries" << endl;
1750 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1751 TIter next(list);
1752 TObjString *str;
1753 while((str=(TObjString*)next())) {
1754 if (str->GetString().Contains(".so"))
1755 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1756 }
1757 if (list) delete list;
1758 }
1759 out << endl;
c57f56b7 1760 out << "// analysis source to be compiled at runtime (if any)" << endl;
1761 if (fAnalysisSource.Length()) {
1762 TObjArray *list = fAnalysisSource.Tokenize(" ");
1763 TIter next(list);
1764 TObjString *str;
1765 while((str=(TObjString*)next())) {
1766 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1767 }
1768 if (list) delete list;
1769 }
1770 out << endl;
1771 out << "// connect to AliEn and make the chain" << endl;
1772 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1773 if (IsUsingTags()) {
1774 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1775 } else {
76535c98 1776 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1777 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1778 } else {
1779 out << " // Check if the macro to create the chain was provided" << endl;
1780 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1781 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1782 out << " return;" << endl;
1783 out << " }" << endl;
1784 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1785 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1786 }
c57f56b7 1787 }
1788 out << "// read the analysis manager from file" << endl;
f10e8481 1789 TString analysisFile = fExecutable;
1790 analysisFile.ReplaceAll(".sh", ".root");
1791 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1792 out << " if (!file) return;" << endl;
1793 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1794 out << " AliAnalysisManager *mgr = 0;" << endl;
1795 out << " TKey *key;" << endl;
1796 out << " while ((key=(TKey*)nextkey())) {" << endl;
1797 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1798 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1799 out << " };" << endl;
1800 out << " if (!mgr) {" << endl;
f10e8481 1801 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1802 out << " return;" << endl;
1803 out << " }" << endl << endl;
1804 out << " mgr->PrintStatus();" << endl;
1805 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1806 out << " timer.Stop();" << endl;
1807 out << " timer.Print();" << endl;
c57f56b7 1808 out << "}" << endl << endl;
1809 if (IsUsingTags()) {
1810 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1811 out << "{" << endl;
1812 out << "// Create a chain using tags from the xml file." << endl;
1813 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1814 out << " if (!coll) {" << endl;
1815 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1816 out << " return NULL;" << endl;
1817 out << " }" << endl;
1818 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1819 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1820 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1821 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1822 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1823 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1824 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1825 out << " // Check if the cuts configuration file was provided" << endl;
1826 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1827 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1828 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1829 out << " }" << endl;
0df6ccf2 1830 if (fFriendChainName=="") {
1831 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1832 } else {
1833 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1834 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1835 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1836 }
c57f56b7 1837 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1838 out << " chain->ls();" << endl;
1839 out << " return chain;" << endl;
fcc9bb6f 1840 out << "}" << endl << endl;
c57f56b7 1841 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1842 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1843 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1844 msg += " AliLHCTagCuts *lhcCuts,\n";
1845 msg += " AliDetectorTagCuts *detCuts,\n";
1846 msg += " AliEventTagCuts *evCuts)";
1847 Info("WriteAnalysisMacro", msg.Data());
1848 }
0df6ccf2 1849 }
1850 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1851 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1852 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1853 out << "{" << endl;
1854 out << "// Create a chain using url's from xml file" << endl;
1855 out << " TString treename = type;" << endl;
1856 out << " treename.ToLower();" << endl;
1857 out << " treename += \"Tree\";" << endl;
e02fee64 1858 out << " printf(\"***************************************\\n\");" << endl;
1859 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1860 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1861 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1862 out << " if (!coll) {" << endl;
1863 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1864 out << " return NULL;" << endl;
1865 out << " }" << endl;
1866 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1867 if(fFriendChainName!="") {
1868 out << " TChain *chainFriend = new TChain(treename);" << endl;
1869 }
c57f56b7 1870 out << " coll->Reset();" << endl;
0df6ccf2 1871 out << " while (coll->Next()) {" << endl;
1872 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1873 if(fFriendChainName!="") {
1874 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1875 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1876 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1877 out << " chainFriend->Add(fileFriend.Data());" << endl;
1878 }
1879 out << " }" << endl;
c57f56b7 1880 out << " if (!chain->GetNtrees()) {" << endl;
1881 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1882 out << " return NULL;" << endl;
1883 out << " }" << endl;
0df6ccf2 1884 if(fFriendChainName!="") {
1885 out << " chain->AddFriend(chainFriend);" << endl;
1886 }
c57f56b7 1887 out << " return chain;" << endl;
fcc9bb6f 1888 out << "}" << endl << endl;
c57f56b7 1889 }
4e5c5506 1890 if (fPackages) {
fcc9bb6f 1891 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1892 out << "Bool_t SetupPar(const char *package) {" << endl;
1893 out << "// Compile the package and set it up." << endl;
1894 out << " TString pkgdir = package;" << endl;
1895 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1896 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1897 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1898 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1899 out << " // Check for BUILD.sh and execute" << endl;
1900 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1901 out << " printf(\"*******************************\\n\");" << endl;
1902 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1903 out << " printf(\"*******************************\\n\");" << endl;
1904 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1905 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1906 out << " gSystem->ChangeDirectory(cdir);" << endl;
1907 out << " return kFALSE;" << endl;
1908 out << " }" << endl;
1909 out << " } else {" << endl;
fcc9bb6f 1910 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1911 out << " gSystem->ChangeDirectory(cdir);" << endl;
1912 out << " return kFALSE;" << endl;
1913 out << " }" << endl;
1914 out << " // Check for SETUP.C and execute" << endl;
1915 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1916 out << " printf(\"*******************************\\n\");" << endl;
1917 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1918 out << " printf(\"*******************************\\n\");" << endl;
1919 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1920 out << " } else {" << endl;
fcc9bb6f 1921 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1922 out << " gSystem->ChangeDirectory(cdir);" << endl;
1923 out << " return kFALSE;" << endl;
1924 out << " }" << endl;
1925 out << " // Restore original workdir" << endl;
1926 out << " gSystem->ChangeDirectory(cdir);" << endl;
1927 out << " return kTRUE;" << endl;
1928 out << "}" << endl;
1929 }
c57f56b7 1930 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1931 }
1932 Bool_t copy = kTRUE;
1933 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1934 if (copy) {
1935 CdWork();
1936 TString workdir = gGrid->GetHomeDirectory();
1937 workdir += fGridWorkingDir;
1938 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1939 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1940 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1941 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1942 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1943 }
1944 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1945 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1946 }
1947}
1948
1949//______________________________________________________________________________
1950void AliAnalysisAlien::WriteExecutable()
1951{
1952// Generate the alien executable script.
1953 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1954 ofstream out;
1955 out.open(fExecutable.Data(), ios::out);
1956 if (out.bad()) {
5513444a 1957 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1958 return;
1959 }
1960 out << "#!/bin/bash" << endl;
1961 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1962 out << "echo \"=========================================\"" << endl;
1963 out << "echo \"############## PATH : ##############\"" << endl;
1964 out << "echo $PATH" << endl;
1965 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1966 out << "echo $LD_LIBRARY_PATH" << endl;
1967 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1968 out << "echo $ROOTSYS" << endl;
1969 out << "echo \"############## which root : ##############\"" << endl;
1970 out << "which root" << endl;
1971 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1972 out << "echo $ALICE_ROOT" << endl;
1973 out << "echo \"############## which aliroot : ##############\"" << endl;
1974 out << "which aliroot" << endl;
1975 out << "echo \"=========================================\"" << endl << endl;
1976// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1977 out << "root -b -q ";
1978 out << fAnalysisMacro.Data() << endl << endl;
1979 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1980 }
1981 Bool_t copy = kTRUE;
1982 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1983 if (copy) {
1984 CdWork();
1985 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 1986 TString bindir = Form("%s/bin", workdir.Data());
1987 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 1988 workdir += fGridWorkingDir;
1989 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1990 if (FileExists(executable)) gGrid->Rm(executable);
1991 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1992 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1993 }
1994}
1995
1996//______________________________________________________________________________
5513444a 1997void AliAnalysisAlien::WriteProductionFile(const char *filename) const
1998{
1999// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2000// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2001// Next lines: full_path_to_dataset XXX (XXX is a string)
2002// To submit, one has to: submit jdl XXX for all lines
2003 ofstream out;
2004 out.open(filename, ios::out);
2005 if (out.bad()) {
2006 Error("WriteProductionFile", "Bad file name: %s", filename);
2007 return;
2008 }
2009 TString workdir = gGrid->GetHomeDirectory();
2010 workdir += fGridWorkingDir;
f5e8c702 2011 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2012 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2013 out << locjdl << " " << njobspermaster << endl;
5513444a 2014 Int_t nmasterjobs = fInputFiles->GetEntries();
2015 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2016 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2017 }
2018 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2019 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2020}
2021
2022//______________________________________________________________________________
c57f56b7 2023void AliAnalysisAlien::WriteValidationScript()
2024{
2025// Generate the alien validation script.
2026 // Generate the validation script
2027 TObjString *os;
0d5d317c 2028 TString validationScript = fExecutable;
2029 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2030 if (!Connect()) {
2031 Error("WriteValidationScript", "Alien connection required");
2032 return;
2033 }
2034 TString out_stream = "";
2035 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2036 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2037 ofstream out;
0d5d317c 2038 out.open(validationScript, ios::out);
c57f56b7 2039 out << "#!/bin/bash" << endl;
2040 out << "##################################################" << endl;
2041 out << "validateout=`dirname $0`" << endl;
2042 out << "validatetime=`date`" << endl;
2043 out << "validated=\"0\";" << endl;
2044 out << "error=0" << endl;
2045 out << "if [ -z $validateout ]" << endl;
2046 out << "then" << endl;
2047 out << " validateout=\".\"" << endl;
2048 out << "fi" << endl << endl;
2049 out << "cd $validateout;" << endl;
2050 out << "validateworkdir=`pwd`;" << endl << endl;
2051 out << "echo \"*******************************************************\"" << out_stream << endl;
2052 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2053 out << "" << endl;
2054 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2055 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2056 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2057 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2058 out << "ls -la ./" << out_stream << endl;
2059 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2060 out << "##################################################" << endl;
ebec370a 2061
2062 out << "" << endl;
2063 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2064 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2065 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2066 out << "" << endl;
2067
2068 out << "if [ ! -f stderr ] ; then" << endl;
2069 out << " error=1" << endl;
2070 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2071 out << " echo \"Error = $error\" " << out_stream << endl;
2072 out << "fi" << endl;
2073
2074 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2075 out << " error=1" << endl;
2076 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2077 out << " echo \"$parArch\" " << out_stream << endl;
2078 out << " echo \"Error = $error\" " << out_stream << endl;
2079 out << "fi" << endl;
2080
2081 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2082 out << " error=1" << endl;
2083 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2084 out << " echo \"$segViol\" " << out_stream << endl;
2085 out << " echo \"Error = $error\" " << out_stream << endl;
2086 out << "fi" << endl;
2087
2088 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2089 out << " error=1" << endl;
2090 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2091 out << " echo \"$segFault\" " << out_stream << endl;
2092 out << " echo \"Error = $error\" " << out_stream << endl;
2093 out << "fi" << endl;
2094
2095 // Part dedicated to the specific analyses running into the train
2096
c57f56b7 2097 TObjArray *arr = fOutputFiles.Tokenize(" ");
2098 TIter next1(arr);
2099 TString output_file;
2100 while ((os=(TObjString*)next1())) {
2101 output_file = os->GetString();
2102 Int_t index = output_file.Index("@");
2103 if (index > 0) output_file.Remove(index);
2104 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2105 out << " error=1" << endl;
2106 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2107 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2108 out << "fi" << endl;
2109 }
2110 delete arr;
923e2ca5 2111 out << "if ! [ -f outputs_valid ] ; then" << endl;
2112 out << " error=1" << endl;
2113 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2114 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2115 out << "fi" << endl;
2116
c57f56b7 2117 out << "if [ $error = 0 ] ; then" << endl;
2118 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2119 out << "fi" << endl;
2120
2121 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2122 out << "echo \"*******************************************************\"" << out_stream << endl;
2123 out << "cd -" << endl;
2124 out << "exit $error" << endl;
2125 }
2126 Bool_t copy = kTRUE;
2127 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2128 if (copy) {
2129 CdWork();
2130 TString workdir = gGrid->GetHomeDirectory();
2131 workdir += fGridWorkingDir;
0d5d317c 2132 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2133 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2134 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2135 }
2136}