]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
For the display, now uses the human chamberId convention, i.e. starting at one, inste...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
59 fArguments(),
60 fAnalysisMacro(),
61 fAnalysisSource(),
62 fAdditionalLibs(),
63 fSplitMode(),
64 fAPIVersion(),
65 fROOTVersion(),
66 fAliROOTVersion(),
67 fUser(),
68 fGridWorkingDir(),
69 fGridDataDir(),
70 fDataPattern(),
71 fGridOutputDir(),
72 fOutputArchive(),
73 fOutputFiles(),
74 fInputFormat(),
e7c71df0 75 fDatasetName(),
c57f56b7 76 fJDLName(),
bb885a9e 77 fMergeExcludes(),
f965131e 78 fIncludePath(),
bb885a9e 79 fCloseSE(),
0df6ccf2 80 fFriendChainName(),
c6cb3634 81 fJobTag(),
4e5c5506 82 fInputFiles(0),
83 fPackages(0)
c57f56b7 84{
85// Dummy ctor.
86 SetDefaults();
87}
88
89//______________________________________________________________________________
90AliAnalysisAlien::AliAnalysisAlien(const char *name)
91 :AliAnalysisGrid(name),
92 fGridJDL(NULL),
93 fPrice(0),
94 fTTL(0),
95 fSplitMaxInputFileNumber(0),
96 fMaxInitFailed(0),
97 fMasterResubmitThreshold(0),
bb885a9e 98 fNtestFiles(0),
319593fb 99 fNrunsPerMaster(0),
16a4353c 100 fMaxMergeFiles(0),
d2a409b2 101 fNsubmitted(0),
a3e84053 102 fProductionMode(0),
c57f56b7 103 fRunNumbers(),
104 fExecutable(),
105 fArguments(),
106 fAnalysisMacro(),
107 fAnalysisSource(),
108 fAdditionalLibs(),
109 fSplitMode(),
110 fAPIVersion(),
111 fROOTVersion(),
112 fAliROOTVersion(),
113 fUser(),
114 fGridWorkingDir(),
115 fGridDataDir(),
116 fDataPattern(),
117 fGridOutputDir(),
118 fOutputArchive(),
119 fOutputFiles(),
120 fInputFormat(),
e7c71df0 121 fDatasetName(),
c57f56b7 122 fJDLName(),
bb885a9e 123 fMergeExcludes(),
f965131e 124 fIncludePath(),
bb885a9e 125 fCloseSE(),
0df6ccf2 126 fFriendChainName(),
c6cb3634 127 fJobTag(),
4e5c5506 128 fInputFiles(0),
129 fPackages(0)
c57f56b7 130{
131// Default ctor.
132 SetDefaults();
133}
134
135//______________________________________________________________________________
136AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
137 :AliAnalysisGrid(other),
138 fGridJDL(NULL),
139 fPrice(other.fPrice),
140 fTTL(other.fTTL),
141 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
142 fMaxInitFailed(other.fMaxInitFailed),
143 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 144 fNtestFiles(other.fNtestFiles),
319593fb 145 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 146 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 147 fNsubmitted(other.fNsubmitted),
a3e84053 148 fProductionMode(other.fProductionMode),
c57f56b7 149 fRunNumbers(other.fRunNumbers),
150 fExecutable(other.fExecutable),
151 fArguments(other.fArguments),
152 fAnalysisMacro(other.fAnalysisMacro),
153 fAnalysisSource(other.fAnalysisSource),
154 fAdditionalLibs(other.fAdditionalLibs),
155 fSplitMode(other.fSplitMode),
156 fAPIVersion(other.fAPIVersion),
157 fROOTVersion(other.fROOTVersion),
158 fAliROOTVersion(other.fAliROOTVersion),
159 fUser(other.fUser),
160 fGridWorkingDir(other.fGridWorkingDir),
161 fGridDataDir(other.fGridDataDir),
162 fDataPattern(other.fDataPattern),
163 fGridOutputDir(other.fGridOutputDir),
164 fOutputArchive(other.fOutputArchive),
165 fOutputFiles(other.fOutputFiles),
166 fInputFormat(other.fInputFormat),
e7c71df0 167 fDatasetName(other.fDatasetName),
c57f56b7 168 fJDLName(other.fJDLName),
bb885a9e 169 fMergeExcludes(other.fMergeExcludes),
f965131e 170 fIncludePath(other.fIncludePath),
bb885a9e 171 fCloseSE(other.fCloseSE),
0df6ccf2 172 fFriendChainName(other.fFriendChainName),
c6cb3634 173 fJobTag(other.fJobTag),
4e5c5506 174 fInputFiles(0),
175 fPackages(0)
c57f56b7 176{
177// Copy ctor.
178 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 179 fRunRange[0] = other.fRunRange[0];
180 fRunRange[1] = other.fRunRange[1];
c57f56b7 181 if (other.fInputFiles) {
182 fInputFiles = new TObjArray();
183 TIter next(other.fInputFiles);
184 TObject *obj;
185 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
186 fInputFiles->SetOwner();
187 }
4e5c5506 188 if (other.fPackages) {
189 fPackages = new TObjArray();
190 TIter next(other.fPackages);
191 TObject *obj;
192 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
193 fPackages->SetOwner();
194 }
c57f56b7 195}
196
197//______________________________________________________________________________
198AliAnalysisAlien::~AliAnalysisAlien()
199{
200// Destructor.
201 if (fGridJDL) delete fGridJDL;
202 if (fInputFiles) delete fInputFiles;
4e5c5506 203 if (fPackages) delete fPackages;
c57f56b7 204}
205
206//______________________________________________________________________________
207AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
208{
209// Assignment.
210 if (this != &other) {
211 AliAnalysisGrid::operator=(other);
212 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
213 fPrice = other.fPrice;
214 fTTL = other.fTTL;
215 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
216 fMaxInitFailed = other.fMaxInitFailed;
217 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 218 fNtestFiles = other.fNtestFiles;
a3e84053 219 fNrunsPerMaster = other.fNrunsPerMaster;
220 fMaxMergeFiles = other.fMaxMergeFiles;
221 fNsubmitted = other.fNsubmitted;
222 fProductionMode = other.fProductionMode;
c57f56b7 223 fRunNumbers = other.fRunNumbers;
224 fExecutable = other.fExecutable;
225 fArguments = other.fArguments;
226 fAnalysisMacro = other.fAnalysisMacro;
227 fAnalysisSource = other.fAnalysisSource;
228 fAdditionalLibs = other.fAdditionalLibs;
229 fSplitMode = other.fSplitMode;
230 fAPIVersion = other.fAPIVersion;
231 fROOTVersion = other.fROOTVersion;
232 fAliROOTVersion = other.fAliROOTVersion;
233 fUser = other.fUser;
234 fGridWorkingDir = other.fGridWorkingDir;
235 fGridDataDir = other.fGridDataDir;
236 fDataPattern = other.fDataPattern;
237 fGridOutputDir = other.fGridOutputDir;
238 fOutputArchive = other.fOutputArchive;
239 fOutputFiles = other.fOutputFiles;
240 fInputFormat = other.fInputFormat;
e7c71df0 241 fDatasetName = other.fDatasetName;
c57f56b7 242 fJDLName = other.fJDLName;
bb885a9e 243 fMergeExcludes = other.fMergeExcludes;
f965131e 244 fIncludePath = other.fIncludePath;
bb885a9e 245 fCloseSE = other.fCloseSE;
0df6ccf2 246 fFriendChainName = other.fFriendChainName;
c6cb3634 247 fJobTag = other.fJobTag;
c57f56b7 248 if (other.fInputFiles) {
249 fInputFiles = new TObjArray();
250 TIter next(other.fInputFiles);
251 TObject *obj;
252 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
253 fInputFiles->SetOwner();
254 }
4e5c5506 255 if (other.fPackages) {
256 fPackages = new TObjArray();
257 TIter next(other.fPackages);
258 TObject *obj;
259 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
260 fPackages->SetOwner();
261 }
c57f56b7 262 }
263 return *this;
264}
265
f965131e 266//______________________________________________________________________________
267void AliAnalysisAlien::AddIncludePath(const char *path)
268{
269// Add include path in the remote analysis macro.
270 TString p(path);
271 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
272 else fIncludePath += Form("-I%s ", path);
273}
274
c57f56b7 275//______________________________________________________________________________
276void AliAnalysisAlien::AddRunNumber(Int_t run)
277{
278// Add a run number to the list of runs to be processed.
279 if (fRunNumbers.Length()) fRunNumbers += " ";
280 fRunNumbers += Form("%d", run);
281}
282
283//______________________________________________________________________________
284void AliAnalysisAlien::AddDataFile(const char *lfn)
285{
286// Adds a data file to the input to be analysed. The file should be a valid LFN
287// or point to an existing file in the alien workdir.
288 if (!fInputFiles) fInputFiles = new TObjArray();
289 fInputFiles->Add(new TObjString(lfn));
290}
291
292//______________________________________________________________________________
293Bool_t AliAnalysisAlien::Connect()
294{
295// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
296 if (gGrid && gGrid->IsConnected()) return kTRUE;
297 if (!gSystem->Getenv("alien_API_USER")) {
298 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
299 gSystem->Getenv("UID"));
300 return kFALSE;
301 }
302 if (!gGrid) {
303 Info("Connect", "Trying to connect to AliEn ...");
304 TGrid::Connect("alien://");
305 }
306 if (!gGrid || !gGrid->IsConnected()) {
307 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
308 return kFALSE;
309 }
310 fUser = gGrid->GetUser();
311 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
312 return kTRUE;
313}
314
315//______________________________________________________________________________
316void AliAnalysisAlien::CdWork()
317{
318// Check validity of alien workspace. Create directory if possible.
319 if (!Connect()) {
320 Error("CdWork", "Alien connection required");
321 return;
322 }
323 TString homedir = gGrid->GetHomeDirectory();
324 TString workdir = homedir + fGridWorkingDir;
325 if (!gGrid->Cd(workdir)) {
326 gGrid->Cd(homedir);
327 if (gGrid->Mkdir(workdir)) {
328 gGrid->Cd(fGridWorkingDir);
329 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
330 } else {
331 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
332 workdir.Data(), homedir.Data());
333 fGridWorkingDir = "";
334 }
335 }
336}
337
338//______________________________________________________________________________
339Bool_t AliAnalysisAlien::CheckInputData()
340{
341// Check validity of input data. If necessary, create xml files.
a8739e8a 342 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 343 if (!fGridDataDir.Length()) {
344 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
345 return kFALSE;
346 }
347 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
348 return kTRUE;
c57f56b7 349 }
350 // Process declared files
351 Bool_t is_collection = kFALSE;
352 Bool_t is_xml = kFALSE;
353 Bool_t use_tags = kFALSE;
354 Bool_t checked = kFALSE;
355 CdWork();
356 TString file;
357 TString workdir = gGrid->GetHomeDirectory();
358 workdir += fGridWorkingDir;
359 if (fInputFiles) {
360 TObjString *objstr;
361 TIter next(fInputFiles);
362 while ((objstr=(TObjString*)next())) {
363 file = workdir;
364 file += "/";
365 file += objstr->GetString();
366 // Store full lfn path
367 if (FileExists(file)) objstr->SetString(file);
368 else {
369 file = objstr->GetName();
370 if (!FileExists(objstr->GetName())) {
371 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
372 objstr->GetName(), workdir.Data());
373 return kFALSE;
374 }
375 }
376 Bool_t iscoll, isxml, usetags;
377 CheckDataType(file, iscoll, isxml, usetags);
378 if (!checked) {
379 checked = kTRUE;
380 is_collection = iscoll;
381 is_xml = isxml;
382 use_tags = usetags;
383 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
384 } else {
385 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
386 Error("CheckInputData", "Some conflict was found in the types of inputs");
387 return kFALSE;
388 }
389 }
390 }
391 }
392 // Process requested run numbers
a8739e8a 393 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 394 // Check validity of alien data directory
395 if (!fGridDataDir.Length()) {
396 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
397 return kFALSE;
398 }
399 if (!gGrid->Cd(fGridDataDir)) {
400 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
401 return kFALSE;
402 }
403 if (is_collection) {
404 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
405 return kFALSE;
406 }
407
408 if (checked && !is_xml) {
409 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
410 return kFALSE;
411 }
412 // Check validity of run number(s)
413 TObjArray *arr;
414 TObjString *os;
319593fb 415 Int_t nruns = 0;
416 TString schunk;
c57f56b7 417 TString path;
418 if (!checked) {
419 checked = kTRUE;
420 use_tags = fDataPattern.Contains("tag");
421 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
422 }
423 if (use_tags != fDataPattern.Contains("tag")) {
424 Error("CheckInputData", "Cannot mix input files using/not using tags");
425 return kFALSE;
426 }
427 if (fRunNumbers.Length()) {
a8739e8a 428 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 429 arr = fRunNumbers.Tokenize(" ");
430 TIter next(arr);
431 while ((os=(TObjString*)next())) {
432 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
433 if (!gGrid->Cd(path)) {
a8739e8a 434 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
435 continue;
c57f56b7 436 }
437 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
438 TString msg = "\n##### file: ";
439 msg += path;
440 msg += " type: xml_collection;";
441 if (use_tags) msg += " using_tags: Yes";
442 else msg += " using_tags: No";
443 Info("CheckDataType", msg.Data());
319593fb 444 if (fNrunsPerMaster<2) {
d2a409b2 445 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 446 } else {
447 nruns++;
448 if (((nruns-1)%fNrunsPerMaster) == 0) {
449 schunk = os->GetString();
450 }
451 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
452 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 453 AddDataFile(schunk);
319593fb 454 }
c57f56b7 455 }
456 delete arr;
a8739e8a 457 } else {
458 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
459 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
460 path = Form("%s/%d ", fGridDataDir.Data(), irun);
461 if (!gGrid->Cd(path)) {
462 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
463 continue;
464 }
465 path = Form("%s/%d.xml", workdir.Data(),irun);
466 TString msg = "\n##### file: ";
467 msg += path;
468 msg += " type: xml_collection;";
469 if (use_tags) msg += " using_tags: Yes";
470 else msg += " using_tags: No";
471 Info("CheckDataType", msg.Data());
319593fb 472 if (fNrunsPerMaster<2) {
d2a409b2 473 AddDataFile(Form("%d.xml",irun));
319593fb 474 } else {
475 nruns++;
476 if (((nruns-1)%fNrunsPerMaster) == 0) {
477 schunk = Form("%d", irun);
478 }
479 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
480 schunk += Form("_%d.xml", irun);
d2a409b2 481 AddDataFile(schunk);
319593fb 482 }
a8739e8a 483 }
c57f56b7 484 }
485 return kTRUE;
486}
487
488//______________________________________________________________________________
489Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
490{
491// Create dataset for the grid data directory + run number.
492 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
493 if (!Connect()) {
494 Error("CreateDataset", "Cannot create dataset with no grid connection");
495 return kFALSE;
496 }
497
498 // Cd workspace
499 CdWork();
500 TString workdir = gGrid->GetHomeDirectory();
501 workdir += fGridWorkingDir;
502
503 // Compose the 'find' command arguments
504 TString command;
505 TString options = "-x collection ";
bb885a9e 506 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 507 TString conditions = "";
508
509 TString file;
510 TString path;
319593fb 511 Int_t nruns = 0;
512 TString schunk;
ab254fd1 513 TGridCollection *cbase=0, *cadd=0;
d2a409b2 514 if (!fRunNumbers.Length() && !fRunRange[0]) {
515 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
516 // Make a single data collection from data directory.
517 path = fGridDataDir;
518 if (!gGrid->Cd(path)) {
519 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
520 return kFALSE;
521 }
522 CdWork();
523 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
524 else file = Form("%s.xml", gSystem->BaseName(path));
525 if (gSystem->AccessPathName(file)) {
526 command = "find ";
527 command += options;
528 command += path;
529 command += " ";
530 command += pattern;
531 command += conditions;
532 printf("command: %s\n", command.Data());
533 TGridResult *res = gGrid->Command(command);
534 if (res) delete res;
535 // Write standard output to file
536 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
537 }
538 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
539 // Copy xml file to alien space
540 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
541 if (!FileExists(file)) {
542 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
543 return kFALSE;
544 }
545 // Update list of files to be processed.
546 }
547 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
548 return kTRUE;
549 }
c57f56b7 550 // Several runs
a8739e8a 551 if (fRunNumbers.Length()) {
552 TObjArray *arr = fRunNumbers.Tokenize(" ");
553 TObjString *os;
554 TIter next(arr);
555 while ((os=(TObjString*)next())) {
556 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
673d4fb3 557 if (!gGrid->Cd(path)) continue;
95e5b448 558 CdWork();
a8739e8a 559 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
560 else file = Form("%s.xml", os->GetString().Data());
319593fb 561 // If local collection file does not exist, create it via 'find' command.
562 if (gSystem->AccessPathName(file)) {
563 command = "find ";
564 command += options;
565 command += path;
566 command += pattern;
567 command += conditions;
568 TGridResult *res = gGrid->Command(command);
569 if (res) delete res;
570 // Write standard output to file
571 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
572 }
a8739e8a 573 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 574 // Check if there is one run per master job.
575 if (fNrunsPerMaster<2) {
576 if (FileExists(file)) {
577 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
578 continue;
579 }
580 // Copy xml file to alien space
581 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
582 if (!FileExists(file)) {
583 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
584 delete arr;
585 return kFALSE;
586 }
587 } else {
588 nruns++;
589 if (((nruns-1)%fNrunsPerMaster) == 0) {
590 schunk = os->GetString();
591 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
592 } else {
593 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
594 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
595 cbase->Add(cadd);
596 delete cadd;
597 }
598 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
599 continue;
600 }
601 schunk += Form("_%s.xml", os->GetString().Data());
602 if (FileExists(schunk)) {
603 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
604 continue;
605 }
606 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
607 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
608 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
609 if (!FileExists(schunk)) {
610 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
611 delete arr;
612 return kFALSE;
613 }
614 }
a8739e8a 615 }
616 delete arr;
617 } else {
618 // Process a full run range.
619 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
620 path = Form("%s/%d ", fGridDataDir.Data(), irun);
673d4fb3 621 if (!gGrid->Cd(path)) continue;
95e5b448 622 CdWork();
a8739e8a 623 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
624 else file = Form("%d.xml", irun);
319593fb 625 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 626 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
627// gGrid->Rm(file);
628 continue;
629 }
319593fb 630 // If local collection file does not exist, create it via 'find' command.
631 if (gSystem->AccessPathName(file)) {
632 command = "find ";
633 command += options;
634 command += path;
635 command += pattern;
636 command += conditions;
637 TGridResult *res = gGrid->Command(command);
638 if (res) delete res;
639 // Write standard output to file
640 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
641 }
a8739e8a 642 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 643 // Check if there is one run per master job.
644 if (fNrunsPerMaster<2) {
645 if (FileExists(file)) {
646 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
647 continue;
648 }
649 // Copy xml file to alien space
650 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
651 if (!FileExists(file)) {
652 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
653 return kFALSE;
654 }
655 } else {
656 nruns++;
95e5b448 657 // Check if the collection for the chunk exist locally.
658 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
659 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
319593fb 660 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
661 if (((nruns-1)%fNrunsPerMaster) == 0) {
662 schunk = Form("%d", irun);
663 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
664 } else {
665 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
666 cbase->Add(cadd);
667 delete cadd;
668 }
669 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
670 continue;
671 }
672 schunk += Form("_%d.xml", irun);
673 if (FileExists(schunk)) {
674 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
675 continue;
676 }
677 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
678 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 679 if (FileExists(schunk)) {
680 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
681 continue;
682 }
319593fb 683 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
684 if (!FileExists(schunk)) {
685 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
686 return kFALSE;
687 }
688 }
c57f56b7 689 }
a8739e8a 690 }
c57f56b7 691 return kTRUE;
692}
693
694//______________________________________________________________________________
695Bool_t AliAnalysisAlien::CreateJDL()
696{
697// Generate a JDL file according to current settings. The name of the file is
698// specified by fJDLName.
699 Bool_t error = kFALSE;
700 TObjArray *arr = 0;
701 Bool_t copy = kTRUE;
702 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
703 Bool_t generate = kTRUE;
704 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
705 if (!Connect()) {
706 Error("CreateJDL", "Alien connection required");
707 return kFALSE;
708 }
709 // Check validity of alien workspace
710 CdWork();
711 TString workdir = gGrid->GetHomeDirectory();
712 workdir += fGridWorkingDir;
713 if (generate) {
714 TObjString *os;
715 if (!fInputFiles) {
716 Error("CreateJDL()", "Define some input files for your analysis.");
717 error = kTRUE;
718 }
719 // Compose list of input files
720 // Check if output files were defined
721 if (!fOutputFiles.Length()) {
722 Error("CreateJDL", "You must define at least one output file");
723 error = kTRUE;
724 }
725 // Check if an output directory was defined and valid
726 if (!fGridOutputDir.Length()) {
727 Error("CreateJDL", "You must define AliEn output directory");
728 error = kTRUE;
729 } else {
d2a409b2 730 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
c57f56b7 731 if (!gGrid->Cd(fGridOutputDir)) {
732 if (gGrid->Mkdir(fGridOutputDir)) {
733 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
734 } else {
735 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 736 // error = kTRUE;
c57f56b7 737 }
738 }
739 gGrid->Cd(workdir);
740 }
741 // Exit if any error up to now
742 if (error) return kFALSE;
743 // Set JDL fields
744 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 745 fGridJDL->SetExecutable(fExecutable);
c57f56b7 746// fGridJDL->SetTTL((UInt_t)fTTL);
747 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
748 if (fMaxInitFailed > 0)
749 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
750 if (fSplitMaxInputFileNumber > 0)
751 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
752 if (fSplitMode.Length())
753 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
754// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
755 if (fAliROOTVersion.Length())
756 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
757 if (fROOTVersion.Length())
758 fGridJDL->AddToPackages("ROOT", fROOTVersion);
759 if (fAPIVersion.Length())
760 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
761 fGridJDL->SetInputDataListFormat(fInputFormat);
762 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 763 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 764 TString analysisFile = fExecutable;
765 analysisFile.ReplaceAll(".sh", ".root");
766 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 767 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
768 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
769 if (fAdditionalLibs.Length()) {
770 arr = fAdditionalLibs.Tokenize(" ");
771 TIter next(arr);
772 while ((os=(TObjString*)next())) {
773 if (os->GetString().Contains(".so")) continue;
774 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
775 }
776 delete arr;
777 }
4e5c5506 778 if (fPackages) {
779 TIter next(fPackages);
780 TObject *obj;
781 while ((obj=next()))
782 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
783 }
c57f56b7 784 if (fOutputArchive.Length()) {
785 arr = fOutputArchive.Tokenize(" ");
786 TIter next(arr);
787 while ((os=(TObjString*)next()))
43da816a 788 if (!os->GetString().Contains("@") && fCloseSE.Length())
789 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
790 else
791 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 792 delete arr;
793 }
c57f56b7 794 arr = fOutputFiles.Tokenize(" ");
795 TIter next(arr);
43da816a 796 while ((os=(TObjString*)next())) {
e1eaf596 797 // Ignore ouputs in jdl that are also in outputarchive
798 TString sout = os->GetString();
799 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
800 if (fOutputArchive.Contains(sout)) continue;
43da816a 801 if (!os->GetString().Contains("@") && fCloseSE.Length())
802 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
803 else
804 fGridJDL->AddToOutputSandbox(os->GetString());
805 }
c57f56b7 806 delete arr;
807// fGridJDL->SetPrice((UInt_t)fPrice);
808 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 809 TString validationScript = fExecutable;
810 validationScript.ReplaceAll(".sh", "_validation.sh");
811 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 812 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 813 // Write a jdl with 2 input parameters: collection name and output dir name.
814 WriteJDL(copy);
c57f56b7 815 }
816 // Copy jdl to grid workspace
a8739e8a 817 if (copy) {
c57f56b7 818 if (fAdditionalLibs.Length()) {
e7c71df0 819 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 820 TObjString *os;
821 TIter next(arr);
822 while ((os=(TObjString*)next())) {
c57f56b7 823 if (os->GetString().Contains(".so")) continue;
4e5c5506 824 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 825 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
826 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
827 }
828 delete arr;
829 }
4e5c5506 830 if (fPackages) {
831 TIter next(fPackages);
832 TObject *obj;
833 while ((obj=next())) {
834 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
835 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
836 }
837 }
c57f56b7 838 }
839 return kTRUE;
840}
841
a8739e8a 842//______________________________________________________________________________
d2a409b2 843Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 844{
845// Writes one or more JDL's corresponding to findex. If findex is negative,
846// all run numbers are considered in one go (jdl). For non-negative indices
847// they correspond to the indices in the array fInputFiles.
848 if (!fInputFiles) return kFALSE;
849 TObjString *os;
a8739e8a 850 TString workdir = gGrid->GetHomeDirectory();
851 workdir += fGridWorkingDir;
d2a409b2 852
853 if (!fRunNumbers.Length() && !fRunRange[0]) {
854 // One jdl with no parameters in case input data is specified by name.
a8739e8a 855 TIter next(fInputFiles);
856 while ((os=(TObjString*)next()))
857 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 858 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 859 } else {
d2a409b2 860 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
861 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 862 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 863 }
864
865
866 // Generate the JDL as a string
867 TString sjdl = fGridJDL->Generate();
868 Int_t index;
869 index = sjdl.Index("Executable");
870 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
871 index = sjdl.Index("Split ");
872 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
873 index = sjdl.Index("SplitMaxInputFileNumber");
874 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
875 index = sjdl.Index("InputDataCollection");
876 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 877 index = sjdl.Index("InputFile");
878 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
879 index = sjdl.Index("InputDataList ");
880 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
881 index = sjdl.Index("InputDataListFormat");
882 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
883 index = sjdl.Index("Price");
884 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
885 index = sjdl.Index("Requirements");
886 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
887 index = sjdl.Index("Packages");
888 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
889 index = sjdl.Index("User =");
890 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
891 index = sjdl.Index("TTL");
892 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
893 index = sjdl.Index("OutputFile");
894 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
895 index = sjdl.Index("OutputDir");
896 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
897 index = sjdl.Index("OutputArchive");
898 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
899 index = sjdl.Index("MaxInitFailed");
900 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
901 index = sjdl.Index("MasterResubmitThreshold");
902 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
903 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
904 index = sjdl.Index("Validationcommand");
905 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
906 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
907 sjdl.ReplaceAll("(member", "\n (member");
908 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
909 sjdl.ReplaceAll("{", "{\n ");
910 sjdl.ReplaceAll("};", "\n};");
911 sjdl.ReplaceAll("{\n \n", "{\n");
912 sjdl.ReplaceAll("\n\n", "\n");
913 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
914 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 915 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 916 index = sjdl.Index("JDLVariables");
917 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
918 // Write jdl to file
a8739e8a 919 ofstream out;
d2a409b2 920 out.open(fJDLName.Data(), ios::out);
a8739e8a 921 if (out.bad()) {
d2a409b2 922 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 923 return kFALSE;
924 }
925 out << sjdl << endl;
926
927 // Copy jdl to grid workspace
928 if (!copy) {
929 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
930 } else {
d2a409b2 931 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
932 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 933 if (fProductionMode)
b5fe9cba 934 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 935 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 936 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 937 }
938 return kTRUE;
939}
940
c57f56b7 941//______________________________________________________________________________
5513444a 942Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 943{
944// Returns true if file exists.
5513444a 945 if (!gGrid) return kFALSE;
c57f56b7 946 TGridResult *res = gGrid->Ls(lfn);
947 if (!res) return kFALSE;
948 TMap *map = dynamic_cast<TMap*>(res->At(0));
949 if (!map) {
950 delete res;
951 return kFALSE;
952 }
953 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
954 if (!objs || !objs->GetString().Length()) {
955 delete res;
956 return kFALSE;
957 }
958 delete res;
959 return kTRUE;
960}
961
962//______________________________________________________________________________
963void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
964{
965// Check input data type.
966 is_collection = kFALSE;
967 is_xml = kFALSE;
968 use_tags = kFALSE;
969 if (!gGrid) {
970 Error("CheckDataType", "No connection to grid");
971 return;
972 }
973 is_collection = IsCollection(lfn);
974 TString msg = "\n##### file: ";
975 msg += lfn;
976 if (is_collection) {
977 msg += " type: raw_collection;";
978 // special treatment for collections
979 is_xml = kFALSE;
980 // check for tag files in the collection
981 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
982 if (!res) {
983 msg += " using_tags: No (unknown)";
984 Info("CheckDataType", msg.Data());
985 return;
986 }
987 const char* typeStr = res->GetKey(0, "origLFN");
988 if (!typeStr || !strlen(typeStr)) {
989 msg += " using_tags: No (unknown)";
990 Info("CheckDataType", msg.Data());
991 return;
992 }
993 TString file = typeStr;
994 use_tags = file.Contains(".tag");
995 if (use_tags) msg += " using_tags: Yes";
996 else msg += " using_tags: No";
997 Info("CheckDataType", msg.Data());
998 return;
999 }
1000 TString slfn(lfn);
1001 slfn.ToLower();
1002 is_xml = slfn.Contains(".xml");
1003 if (is_xml) {
1004 // Open xml collection and check if there are tag files inside
1005 msg += " type: xml_collection;";
1006 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1007 if (!coll) {
1008 msg += " using_tags: No (unknown)";
1009 Info("CheckDataType", msg.Data());
1010 return;
1011 }
1012 TMap *map = coll->Next();
1013 if (!map) {
1014 msg += " using_tags: No (unknown)";
1015 Info("CheckDataType", msg.Data());
1016 return;
1017 }
1018 map = (TMap*)map->GetValue("");
1019 TString file;
1020 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1021 use_tags = file.Contains(".tag");
1022 delete coll;
1023 if (use_tags) msg += " using_tags: Yes";
1024 else msg += " using_tags: No";
1025 Info("CheckDataType", msg.Data());
1026 return;
1027 }
1028 use_tags = slfn.Contains(".tag");
1029 if (slfn.Contains(".root")) msg += " type: root file;";
1030 else msg += " type: unhnown file;";
1031 if (use_tags) msg += " using_tags: Yes";
1032 else msg += " using_tags: No";
1033 Info("CheckDataType", msg.Data());
1034}
1035
4e5c5506 1036//______________________________________________________________________________
1037void AliAnalysisAlien::EnablePackage(const char *package)
1038{
1039// Enables a par file supposed to exist in the current directory.
1040 TString pkg(package);
1041 pkg.ReplaceAll(".par", "");
1042 pkg += ".par";
1043 if (gSystem->AccessPathName(pkg)) {
1044 Error("EnablePackage", "Package %s not found", pkg.Data());
1045 return;
1046 }
1047 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1048 Info("EnablePackage", "AliEn plugin will use .par packages");
1049 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1050 if (!fPackages) {
1051 fPackages = new TObjArray();
1052 fPackages->SetOwner();
1053 }
1054 fPackages->Add(new TObjString(pkg));
1055}
1056
d2a409b2 1057//______________________________________________________________________________
1058const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1059{
1060// Get job status for all jobs with jobid>jobidstart.
1061 static char mstatus[20];
1062 mstatus[0] = '\0';
1063 nrunning = 0;
1064 nwaiting = 0;
1065 nerror = 0;
1066 ndone = 0;
1067 TGridJobStatusList *list = gGrid->Ps("");
1068 if (!list) return mstatus;
1069 Int_t nentries = list->GetSize();
1070 TGridJobStatus *status;
1071 Int_t pid;
1072 for (Int_t ijob=0; ijob<nentries; ijob++) {
1073 status = (TGridJobStatus *)list->At(ijob);
1074 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1075 if (pid<jobidstart) continue;
1076 if (pid == lastid) {
1077 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1078 }
1079 switch (status->GetStatus()) {
1080 case TGridJobStatus::kWAITING:
1081 nwaiting++; break;
1082 case TGridJobStatus::kRUNNING:
1083 nrunning++; break;
1084 case TGridJobStatus::kABORTED:
1085 case TGridJobStatus::kFAIL:
1086 case TGridJobStatus::kUNKNOWN:
1087 nerror++; break;
1088 case TGridJobStatus::kDONE:
1089 ndone++;
1090 }
1091 }
1092 list->Delete();
1093 delete list;
1094 return mstatus;
1095}
1096
c57f56b7 1097//______________________________________________________________________________
1098Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1099{
1100// Returns true if file is a collection. Functionality duplicated from
1101// TAlien::Type() because we don't want to directly depend on TAlien.
1102 if (!gGrid) {
1103 Error("IsCollection", "No connection to grid");
1104 return kFALSE;
1105 }
1106 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1107 if (!res) return kFALSE;
1108 const char* typeStr = res->GetKey(0, "type");
1109 if (!typeStr || !strlen(typeStr)) return kFALSE;
1110 if (!strcmp(typeStr, "collection")) return kTRUE;
1111 delete res;
1112 return kFALSE;
1113}
1114
16a4353c 1115//______________________________________________________________________________
1116void AliAnalysisAlien::Print(Option_t *) const
1117{
1118// Print current plugin settings.
1119 printf("### AliEn analysis plugin current settings ###\n");
a3e84053 1120 printf("= Production mode:______________________________ %d\n", fProductionMode);
16a4353c 1121 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1122 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1123 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1124 if (fUser.Length())
1125 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1126 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1127 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1128 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1129 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1130 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1131 if (fRunNumbers.Length())
1132 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1133 if (fRunRange[0])
1134 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1135 if (!fRunRange[0] && !fRunNumbers.Length()) {
1136 TIter next(fInputFiles);
1137 TObject *obj;
1138 TString list;
1139 while ((obj=next())) list += obj->GetName();
1140 printf("= Input files to be processed: _________________ %s\n", list.Data());
1141 }
1142 if (TestBit(AliAnalysisGrid::kTest))
1143 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1144 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1145 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1146 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1147 printf("=====================================================================\n");
1148 printf("= Job price: ___________________________________ %d\n", fPrice);
1149 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1150 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1151 if (fMaxInitFailed>0)
1152 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1153 if (fMasterResubmitThreshold>0)
1154 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1155 if (fNrunsPerMaster>0)
1156 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
16a4353c 1157 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1158 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
1159 if (fArguments.Length())
1160 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1161 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1162 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1163 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1164 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1165 if (fDatasetName)
1166 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1167 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1168 if (fIncludePath.Data())
1169 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1170 if (fCloseSE.Length())
1171 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1172 if (fFriendChainName.Length())
1173 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1174 if (fPackages) {
1175 TIter next(fPackages);
1176 TObject *obj;
1177 TString list;
1178 while ((obj=next())) list += obj->GetName();
1179 printf("= Par files to be used: ________________________ %s\n", list.Data());
1180 }
1181}
1182
c57f56b7 1183//______________________________________________________________________________
1184void AliAnalysisAlien::SetDefaults()
1185{
1186// Set default values for everything. What cannot be filled will be left empty.
1187 if (fGridJDL) delete fGridJDL;
1188 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1189 fPrice = 1;
1190 fTTL = 30000;
1191 fSplitMaxInputFileNumber = 100;
1192 fMaxInitFailed = 0;
1193 fMasterResubmitThreshold = 0;
bb885a9e 1194 fNtestFiles = 10;
a8739e8a 1195 fRunRange[0] = 0;
1196 fRunRange[1] = 0;
319593fb 1197 fNrunsPerMaster = 1;
16a4353c 1198 fMaxMergeFiles = 100;
c57f56b7 1199 fRunNumbers = "";
1200 fExecutable = "analysis.sh";
1201 fArguments = "";
1202 fAnalysisMacro = "myAnalysis.C";
1203 fAnalysisSource = "";
1204 fAdditionalLibs = "";
1205 fSplitMode = "se";
1206 fAPIVersion = "";
1207 fROOTVersion = "";
1208 fAliROOTVersion = "";
1209 fUser = ""; // Your alien user name
1210 fGridWorkingDir = "";
1211 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1212 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1213 fFriendChainName = "";
c57f56b7 1214 fGridOutputDir = "output";
1215 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1216 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1217 fInputFormat = "xml-single";
1218 fJDLName = "analysis.jdl";
c6cb3634 1219 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1220 fMergeExcludes = "";
c57f56b7 1221}
1222
1223//______________________________________________________________________________
1224Bool_t AliAnalysisAlien::MergeOutputs()
1225{
1226// Merge analysis outputs existing in the AliEn space.
1227 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1228 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1229 if (!Connect()) {
1230 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1231 return kFALSE;
1232 }
1233 // Get the output path
d2a409b2 1234 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1235 if (!gGrid->Cd(fGridOutputDir)) {
c57f56b7 1236 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1237 return kFALSE;
1238 }
1239 if (!fOutputFiles.Length()) {
1240 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1241 return kFALSE;
1242 }
1243 TObjArray *list = fOutputFiles.Tokenize(" ");
1244 TIter next(list);
1245 TObjString *str;
1246 TString command;
1247 TString output_file;
16a4353c 1248 TString output_chunk;
1249 TString previous_chunk;
1250 Int_t count_chunk = 0;
1251 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1252 Bool_t merged = kTRUE;
1253 while((str=(TObjString*)next())) {
1254 output_file = str->GetString();
1255 Int_t index = output_file.Index("@");
1256 if (index > 0) output_file.Remove(index);
319593fb 1257 // Skip already merged outputs
1258 if (!gSystem->AccessPathName(output_file)) {
1259 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1260 continue;
1261 }
bb885a9e 1262 if (fMergeExcludes.Length() &&
1263 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1264 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1265 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
c57f56b7 1266 printf("command: %s\n", command.Data());
1267 TGridResult *res = gGrid->Command(command);
1268 if (!res) continue;
1269 TFileMerger *fm = 0;
1270 TIter nextmap(res);
ab254fd1 1271 TMap *map = 0;
16a4353c 1272 previous_chunk = "";
1273 count_chunk = 0;
319593fb 1274 // Check if there is a merge operation to resume
1275 output_chunk = output_file;
1276 output_chunk.ReplaceAll(".root", "_*.root");
1277 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1278 while (1) {
1279 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1280 if (!map) {
1281 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1282 delete res;
1283 return kFALSE;
1284 }
1285 output_chunk = output_file;
1286 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1287 printf("%s\n", output_chunk.Data());
1288 count_chunk++;
1289 if (gSystem->AccessPathName(output_chunk)) continue;
1290 // Merged file with chunks up to <count_chunk> found
1291 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1292 previous_chunk = output_chunk;
1293 break;
1294 }
1295 }
16a4353c 1296 count_zero = fMaxMergeFiles;
c57f56b7 1297 while ((map=(TMap*)nextmap())) {
16a4353c 1298 // Loop 'find' results and get next LFN
1299 if (count_zero == fMaxMergeFiles) {
1300 // First file in chunk - create file merger and add previous chunk if any.
1301 fm = new TFileMerger(kFALSE);
1302 fm->SetFastMethod(kTRUE);
1303 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1304 output_chunk = output_file;
1305 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1306 }
1307 // If last file found, put merged results in the output file
1308 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1309 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1310 if (!objs || !objs->GetString().Length()) {
16a4353c 1311 // Nothing found - skip this output
c57f56b7 1312 delete res;
16a4353c 1313 delete fm;
1314 break;
1315 }
1316 // Add file to be merged and decrement chunk counter.
1317 fm->AddFile(objs->GetString());
1318 count_zero--;
1319 if (count_zero==0 || map == res->Last()) {
1320 fm->OutputFile(output_chunk);
1321 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1322 // Nothing found - skip this output
1323 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1324 delete res;
1325 delete fm;
1326 break;
1327 }
1328 // Merge the outputs, then go to next chunk
1329 if (!fm->Merge()) {
1330 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1331 delete res;
1332 delete fm;
1333 merged = kFALSE;
1334 break;
1335 } else {
1336 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1337 gSystem->Unlink(previous_chunk);
16a4353c 1338 }
1339 if (map == res->Last()) {
1340 delete res;
1341 delete fm;
1342 break;
1343 }
1344 count_chunk++;
1345 count_zero = fMaxMergeFiles;
1346 previous_chunk = output_chunk;
c57f56b7 1347 }
c57f56b7 1348 }
c57f56b7 1349 }
1350 if (!merged) {
1351 Error("MergeOutputs", "Terminate() will NOT be executed");
1352 }
1353 return merged;
1354}
1355
bb885a9e 1356//______________________________________________________________________________
1357void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1358{
1359// Use the output files connected to output containers from the analysis manager
1360// rather than the files defined by SetOutputFiles
1361 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1362 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1363 analysis manager");
1364 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1365}
1366
5513444a 1367//______________________________________________________________________________
1368Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1369{
1370// Start remote grid analysis.
1371
43da816a 1372 // Check if output files have to be taken from the analysis manager
1373 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1374 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1375 if (!mgr || !mgr->IsInitialized()) {
1376 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1377 return kFALSE;
43da816a 1378 }
1379 fOutputFiles = "";
1380 TIter next(mgr->GetOutputs());
1381 AliAnalysisDataContainer *output;
1382 while ((output=(AliAnalysisDataContainer*)next())) {
1383 const char *filename = output->GetFileName();
1384 if (!(strcmp(filename, "default"))) {
1385 if (!mgr->GetOutputEventHandler()) continue;
1386 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1387 }
1388 if (fOutputFiles.Length()) fOutputFiles += " ";
1389 fOutputFiles += filename;
1390 }
c07b9ce2 1391 // Add extra files registered to the analysis manager
1392 if (mgr->GetExtraFiles().Length()) {
1393 if (fOutputFiles.Length()) fOutputFiles += " ";
1394 fOutputFiles += mgr->GetExtraFiles();
1395 }
43da816a 1396 }
f7b1cbc2 1397// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1398 if (TestBit(AliAnalysisGrid::kOffline)) {
1399 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1400 \n there nor any job run. You can revise the JDL and analysis \
1401 \n macro then run the same in \"submit\" mode.");
1402 } else if (TestBit(AliAnalysisGrid::kTest)) {
1403 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1404 \n dataset.");
1405 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1406 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1407 \n space and job submitted.");
1408 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1409 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1410 return kTRUE;
c57f56b7 1411 } else {
1412 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1413 }
1414
1415 if (!Connect()) {
1416 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1417 return kFALSE;
16a4353c 1418 }
1419 Print();
c57f56b7 1420 if (!CheckInputData()) {
1421 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1422 return kFALSE;
c57f56b7 1423 }
1424 CreateDataset(fDataPattern);
1425 WriteAnalysisFile();
1426 WriteAnalysisMacro();
1427 WriteExecutable();
1428 WriteValidationScript();
5513444a 1429 if (!CreateJDL()) return kFALSE;
1430 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1431 if (TestBit(AliAnalysisGrid::kTest)) {
1432 // Locally testing the analysis
1433 Info("StartAnalysis", "\n_______________________________________________________________________ \
1434 \n Running analysis script in a daughter shell as on a worker node \
1435 \n_______________________________________________________________________");
1436 TObjArray *list = fOutputFiles.Tokenize(" ");
1437 TIter next(list);
1438 TObjString *str;
1439 TString output_file;
1440 while((str=(TObjString*)next())) {
1441 output_file = str->GetString();
1442 Int_t index = output_file.Index("@");
1443 if (index > 0) output_file.Remove(index);
43da816a 1444 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1445 }
1446 delete list;
1447 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1448 TString validationScript = fExecutable;
1449 validationScript.ReplaceAll(".sh", "_validation.sh");
1450 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1451// gSystem->Exec("cat stdout");
5513444a 1452 return kFALSE;
c57f56b7 1453 }
5513444a 1454 // Check if submitting is managed by LPM manager
a3e84053 1455 if (fProductionMode) {
5513444a 1456 TString prodfile = fJDLName;
1457 prodfile.ReplaceAll(".jdl", ".prod");
1458 WriteProductionFile(prodfile);
1459 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1460 return kFALSE;
1461 }
a8739e8a 1462 // Submit AliEn job(s)
d2a409b2 1463 gGrid->Cd(fGridOutputDir);
a8739e8a 1464 TGridResult *res;
c57f56b7 1465 TString jobID = "";
d2a409b2 1466 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1467 // Submit a given xml or a set of runs
a8739e8a 1468 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1469 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1470 if (res) {
1471 const char *cjobId = res->GetKey(0,"jobId");
1472 if (!cjobId) {
1473 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1474 return kFALSE;
a8739e8a 1475 } else {
1476 Info("StartAnalysis", "\n_______________________________________________________________________ \
1477 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1478 \n_______________________________________________________________________",
1479 fJDLName.Data(), cjobId);
1480 jobID = cjobId;
1481 }
1482 delete res;
1483 }
1484 } else {
d2a409b2 1485 // Submit for a range of enumeration of runs.
1486 Submit();
c57f56b7 1487 }
a8739e8a 1488
c57f56b7 1489 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1490 \n You may exit at any time and terminate the job later using the option <terminate> \
1491 \n ##################################################################################", jobID.Data());
bb885a9e 1492 gSystem->Exec("aliensh");
5513444a 1493 return kTRUE;
c57f56b7 1494}
1495
d2a409b2 1496//______________________________________________________________________________
1497void AliAnalysisAlien::Submit()
1498{
1499// Submit all master jobs.
1500 Int_t nmasterjobs = fInputFiles->GetEntries();
1501 Long_t tshoot = gSystem->Now();
1502 if (!fNsubmitted) SubmitNext();
1503 while (fNsubmitted < nmasterjobs) {
1504 Long_t now = gSystem->Now();
1505 if ((now-tshoot)>30000) {
1506 tshoot = now;
1507 SubmitNext();
1508 }
1509 }
1510}
1511
1512//______________________________________________________________________________
1513void AliAnalysisAlien::SubmitNext()
1514{
1515// Submit next bunch of master jobs if the queue is free.
1516 static Bool_t iscalled = kFALSE;
1517 static Int_t firstmaster = 0;
1518 static Int_t lastmaster = 0;
1519 static Int_t npermaster = 0;
1520 if (iscalled) return;
1521 iscalled = kTRUE;
1522 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1523 Int_t ntosubmit = 0;
1524 TGridResult *res;
1525 TString jobID = "";
1526 if (!fNsubmitted) ntosubmit = 1;
1527 else {
1528 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
1529 printf("=== master %d: %s\n", lastmaster, status.Data());
1530 // If last master not split, just return
1531 if (status != "SPLIT") {iscalled = kFALSE; return;}
1532 // No more than 100 waiting jobs
1533 if (nwaiting>100) {iscalled = kFALSE; return;}
1534 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1535 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
1536 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
1537 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1538 }
1539 Int_t nmasterjobs = fInputFiles->GetEntries();
1540 for (Int_t i=0; i<ntosubmit; i++) {
1541 // Submit for a range of enumeration of runs.
1542 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1543 TString query;
1544 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
1545 printf("********* %s\n",query.Data());
1546 res = gGrid->Command(query);
1547 if (res) {
98ca124f 1548 TString cjobId1 = res->GetKey(0,"jobId");
1549 if (!cjobId1.Length()) {
d2a409b2 1550 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1551 iscalled = kFALSE;
1552 return;
1553 } else {
1554 Info("StartAnalysis", "\n_______________________________________________________________________ \
1555 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1556 \n_______________________________________________________________________",
98ca124f 1557 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1558 jobID += cjobId1;
1559 jobID += " ";
98ca124f 1560 lastmaster = cjobId1.Atoi();
d2a409b2 1561 if (!firstmaster) firstmaster = lastmaster;
1562 fNsubmitted++;
1563 }
1564 delete res;
1565 }
1566 }
1567 iscalled = kFALSE;
1568}
1569
c57f56b7 1570//______________________________________________________________________________
1571void AliAnalysisAlien::WriteAnalysisFile()
1572{
f10e8481 1573// Write current analysis manager into the file <analysisFile>
1574 TString analysisFile = fExecutable;
1575 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1576 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1577 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1578 if (!mgr || !mgr->IsInitialized()) {
1579 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1580 return;
1581 }
1582 // Check analysis type
1583 TObject *handler;
1584 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1585 handler = (TObject*)mgr->GetInputEventHandler();
1586 if (handler) {
1587 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1588 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1589 }
1590 TDirectory *cdir = gDirectory;
f10e8481 1591 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1592 if (file) {
1593 mgr->Write();
1594 delete file;
1595 }
1596 if (cdir) cdir->cd();
f10e8481 1597 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1598 }
1599 Bool_t copy = kTRUE;
1600 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1601 if (copy) {
1602 CdWork();
1603 TString workdir = gGrid->GetHomeDirectory();
1604 workdir += fGridWorkingDir;
f10e8481 1605 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1606 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1607 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1608 }
1609}
1610
1611//______________________________________________________________________________
1612void AliAnalysisAlien::WriteAnalysisMacro()
1613{
1614// Write the analysis macro that will steer the analysis in grid mode.
1615 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1616 ofstream out;
1617 out.open(fAnalysisMacro.Data(), ios::out);
1618 if (!out.good()) {
1619 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1620 return;
1621 }
1622 TString func = fAnalysisMacro;
1623 TString type = "ESD";
1624 TString comment = "// Analysis using ";
1625 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1626 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1627 type = "AOD";
1628 comment += "AOD";
1629 }
0df6ccf2 1630 if (type!="AOD" && fFriendChainName!="") {
1631 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1632 return;
1633 }
c57f56b7 1634 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1635 else comment += " data";
1636 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1637 func.ReplaceAll(".C", "");
1638 out << "void " << func.Data() << "()" << endl;
1639 out << "{" << endl;
1640 out << comment.Data() << endl;
1641 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1642 out << " TStopwatch timer;" << endl;
1643 out << " timer.Start();" << endl << endl;
c57f56b7 1644 out << "// load base root libraries" << endl;
1645 out << " gSystem->Load(\"libTree\");" << endl;
1646 out << " gSystem->Load(\"libGeom\");" << endl;
1647 out << " gSystem->Load(\"libVMC\");" << endl;
1648 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1649 out << "// Load analysis framework libraries" << endl;
4e5c5506 1650 if (!fPackages) {
4e5c5506 1651 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1652 out << " gSystem->Load(\"libESD\");" << endl;
1653 out << " gSystem->Load(\"libAOD\");" << endl;
1654 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1655 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1656 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1657 } else {
4e5c5506 1658 TIter next(fPackages);
1659 TObject *obj;
57377eb5 1660 TString pkgname;
1661 Bool_t hasSTEERBase = kFALSE;
1662 Bool_t hasESD = kFALSE;
1663 Bool_t hasAOD = kFALSE;
1664 Bool_t hasANALYSIS = kFALSE;
1665 Bool_t hasANALYSISalice = kFALSE;
1666 Bool_t hasCORRFW = kFALSE;
1667 while ((obj=next())) {
1668 pkgname = obj->GetName();
4478e6f1 1669 if (pkgname == "STEERBase" ||
1670 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1671 if (pkgname == "ESD" ||
1672 pkgname == "ESD.par") hasESD = kTRUE;
1673 if (pkgname == "AOD" ||
1674 pkgname == "AOD.par") hasAOD = kTRUE;
1675 if (pkgname == "ANALYSIS" ||
1676 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1677 if (pkgname == "ANALYSISalice" ||
1678 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1679 if (pkgname == "CORRFW" ||
1680 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1681 }
1682 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1683 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1684 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1685 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1686 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1687 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1688 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1689 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1690 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1691 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1692 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1693 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1694 out << "// Compile other par packages" << endl;
1695 next.Reset();
fcc9bb6f 1696 while ((obj=next())) {
1697 pkgname = obj->GetName();
4478e6f1 1698 if (pkgname == "STEERBase" ||
1699 pkgname == "STEERBase.par" ||
1700 pkgname == "ESD" ||
1701 pkgname == "ESD.par" ||
1702 pkgname == "AOD" ||
1703 pkgname == "AOD.par" ||
1704 pkgname == "ANALYSIS" ||
1705 pkgname == "ANALYSIS.par" ||
1706 pkgname == "ANALYSISalice" ||
1707 pkgname == "ANALYSISalice.par" ||
1708 pkgname == "CORRFW" ||
1709 pkgname == "CORRFW.par") continue;
4e5c5506 1710 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1711 }
4e5c5506 1712 }
43da816a 1713 out << "// include path" << endl;
1714 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1715 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1716 if (fAdditionalLibs.Length()) {
1717 out << "// Add aditional AliRoot libraries" << endl;
1718 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1719 TIter next(list);
1720 TObjString *str;
1721 while((str=(TObjString*)next())) {
1722 if (str->GetString().Contains(".so"))
1723 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1724 }
1725 if (list) delete list;
1726 }
1727 out << endl;
c57f56b7 1728 out << "// analysis source to be compiled at runtime (if any)" << endl;
1729 if (fAnalysisSource.Length()) {
1730 TObjArray *list = fAnalysisSource.Tokenize(" ");
1731 TIter next(list);
1732 TObjString *str;
1733 while((str=(TObjString*)next())) {
1734 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1735 }
1736 if (list) delete list;
1737 }
1738 out << endl;
1739 out << "// connect to AliEn and make the chain" << endl;
1740 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1741 if (IsUsingTags()) {
1742 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1743 } else {
76535c98 1744 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1745 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1746 } else {
1747 out << " // Check if the macro to create the chain was provided" << endl;
1748 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1749 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1750 out << " return;" << endl;
1751 out << " }" << endl;
1752 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1753 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1754 }
c57f56b7 1755 }
1756 out << "// read the analysis manager from file" << endl;
f10e8481 1757 TString analysisFile = fExecutable;
1758 analysisFile.ReplaceAll(".sh", ".root");
1759 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1760 out << " if (!file) return;" << endl;
1761 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1762 out << " AliAnalysisManager *mgr = 0;" << endl;
1763 out << " TKey *key;" << endl;
1764 out << " while ((key=(TKey*)nextkey())) {" << endl;
1765 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1766 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1767 out << " };" << endl;
1768 out << " if (!mgr) {" << endl;
f10e8481 1769 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1770 out << " return;" << endl;
1771 out << " }" << endl << endl;
1772 out << " mgr->PrintStatus();" << endl;
1773 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1774 out << " timer.Stop();" << endl;
1775 out << " timer.Print();" << endl;
c57f56b7 1776 out << "}" << endl << endl;
1777 if (IsUsingTags()) {
1778 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1779 out << "{" << endl;
1780 out << "// Create a chain using tags from the xml file." << endl;
1781 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1782 out << " if (!coll) {" << endl;
1783 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1784 out << " return NULL;" << endl;
1785 out << " }" << endl;
1786 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1787 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1788 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1789 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1790 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1791 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1792 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1793 out << " // Check if the cuts configuration file was provided" << endl;
1794 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1795 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1796 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1797 out << " }" << endl;
0df6ccf2 1798 if (fFriendChainName=="") {
1799 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1800 } else {
1801 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1802 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1803 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1804 }
c57f56b7 1805 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1806 out << " chain->ls();" << endl;
1807 out << " return chain;" << endl;
fcc9bb6f 1808 out << "}" << endl << endl;
c57f56b7 1809 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1810 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1811 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1812 msg += " AliLHCTagCuts *lhcCuts,\n";
1813 msg += " AliDetectorTagCuts *detCuts,\n";
1814 msg += " AliEventTagCuts *evCuts)";
1815 Info("WriteAnalysisMacro", msg.Data());
1816 }
0df6ccf2 1817 }
1818 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1819 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1820 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1821 out << "{" << endl;
1822 out << "// Create a chain using url's from xml file" << endl;
1823 out << " TString treename = type;" << endl;
1824 out << " treename.ToLower();" << endl;
1825 out << " treename += \"Tree\";" << endl;
e02fee64 1826 out << " printf(\"***************************************\\n\");" << endl;
1827 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1828 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1829 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1830 out << " if (!coll) {" << endl;
1831 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1832 out << " return NULL;" << endl;
1833 out << " }" << endl;
1834 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1835 if(fFriendChainName!="") {
1836 out << " TChain *chainFriend = new TChain(treename);" << endl;
1837 }
c57f56b7 1838 out << " coll->Reset();" << endl;
0df6ccf2 1839 out << " while (coll->Next()) {" << endl;
1840 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1841 if(fFriendChainName!="") {
1842 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1843 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1844 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1845 out << " chainFriend->Add(fileFriend.Data());" << endl;
1846 }
1847 out << " }" << endl;
c57f56b7 1848 out << " if (!chain->GetNtrees()) {" << endl;
1849 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1850 out << " return NULL;" << endl;
1851 out << " }" << endl;
0df6ccf2 1852 if(fFriendChainName!="") {
1853 out << " chain->AddFriend(chainFriend);" << endl;
1854 }
c57f56b7 1855 out << " return chain;" << endl;
fcc9bb6f 1856 out << "}" << endl << endl;
c57f56b7 1857 }
4e5c5506 1858 if (fPackages) {
fcc9bb6f 1859 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1860 out << "Bool_t SetupPar(const char *package) {" << endl;
1861 out << "// Compile the package and set it up." << endl;
1862 out << " TString pkgdir = package;" << endl;
1863 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1864 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1865 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1866 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1867 out << " // Check for BUILD.sh and execute" << endl;
1868 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1869 out << " printf(\"*******************************\\n\");" << endl;
1870 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1871 out << " printf(\"*******************************\\n\");" << endl;
1872 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1873 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1874 out << " gSystem->ChangeDirectory(cdir);" << endl;
1875 out << " return kFALSE;" << endl;
1876 out << " }" << endl;
1877 out << " } else {" << endl;
fcc9bb6f 1878 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1879 out << " gSystem->ChangeDirectory(cdir);" << endl;
1880 out << " return kFALSE;" << endl;
1881 out << " }" << endl;
1882 out << " // Check for SETUP.C and execute" << endl;
1883 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1884 out << " printf(\"*******************************\\n\");" << endl;
1885 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1886 out << " printf(\"*******************************\\n\");" << endl;
1887 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1888 out << " } else {" << endl;
fcc9bb6f 1889 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1890 out << " gSystem->ChangeDirectory(cdir);" << endl;
1891 out << " return kFALSE;" << endl;
1892 out << " }" << endl;
1893 out << " // Restore original workdir" << endl;
1894 out << " gSystem->ChangeDirectory(cdir);" << endl;
1895 out << " return kTRUE;" << endl;
1896 out << "}" << endl;
1897 }
c57f56b7 1898 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1899 }
1900 Bool_t copy = kTRUE;
1901 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1902 if (copy) {
1903 CdWork();
1904 TString workdir = gGrid->GetHomeDirectory();
1905 workdir += fGridWorkingDir;
1906 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1907 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1908 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1909 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1910 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1911 }
1912 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1913 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1914 }
1915}
1916
1917//______________________________________________________________________________
1918void AliAnalysisAlien::WriteExecutable()
1919{
1920// Generate the alien executable script.
1921 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1922 ofstream out;
1923 out.open(fExecutable.Data(), ios::out);
1924 if (out.bad()) {
5513444a 1925 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1926 return;
1927 }
1928 out << "#!/bin/bash" << endl;
1929 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1930 out << "echo \"=========================================\"" << endl;
1931 out << "echo \"############## PATH : ##############\"" << endl;
1932 out << "echo $PATH" << endl;
1933 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1934 out << "echo $LD_LIBRARY_PATH" << endl;
1935 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1936 out << "echo $ROOTSYS" << endl;
1937 out << "echo \"############## which root : ##############\"" << endl;
1938 out << "which root" << endl;
1939 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1940 out << "echo $ALICE_ROOT" << endl;
1941 out << "echo \"############## which aliroot : ##############\"" << endl;
1942 out << "which aliroot" << endl;
1943 out << "echo \"=========================================\"" << endl << endl;
1944// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1945 out << "root -b -q ";
1946 out << fAnalysisMacro.Data() << endl << endl;
1947 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1948 }
1949 Bool_t copy = kTRUE;
1950 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1951 if (copy) {
1952 CdWork();
1953 TString workdir = gGrid->GetHomeDirectory();
1954 workdir += fGridWorkingDir;
1955 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1956 if (FileExists(executable)) gGrid->Rm(executable);
1957 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1958 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1959 }
1960}
1961
1962//______________________________________________________________________________
5513444a 1963void AliAnalysisAlien::WriteProductionFile(const char *filename) const
1964{
1965// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 1966// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 1967// Next lines: full_path_to_dataset XXX (XXX is a string)
1968// To submit, one has to: submit jdl XXX for all lines
1969 ofstream out;
1970 out.open(filename, ios::out);
1971 if (out.bad()) {
1972 Error("WriteProductionFile", "Bad file name: %s", filename);
1973 return;
1974 }
1975 TString workdir = gGrid->GetHomeDirectory();
1976 workdir += fGridWorkingDir;
f5e8c702 1977 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 1978 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 1979 out << locjdl << " " << njobspermaster << endl;
5513444a 1980 Int_t nmasterjobs = fInputFiles->GetEntries();
1981 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 1982 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 1983 }
1984 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
1985 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
1986}
1987
1988//______________________________________________________________________________
c57f56b7 1989void AliAnalysisAlien::WriteValidationScript()
1990{
1991// Generate the alien validation script.
1992 // Generate the validation script
1993 TObjString *os;
0d5d317c 1994 TString validationScript = fExecutable;
1995 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 1996 if (!Connect()) {
1997 Error("WriteValidationScript", "Alien connection required");
1998 return;
1999 }
2000 TString out_stream = "";
2001 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2002 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2003 ofstream out;
0d5d317c 2004 out.open(validationScript, ios::out);
c57f56b7 2005 out << "#!/bin/bash" << endl;
2006 out << "##################################################" << endl;
2007 out << "validateout=`dirname $0`" << endl;
2008 out << "validatetime=`date`" << endl;
2009 out << "validated=\"0\";" << endl;
2010 out << "error=0" << endl;
2011 out << "if [ -z $validateout ]" << endl;
2012 out << "then" << endl;
2013 out << " validateout=\".\"" << endl;
2014 out << "fi" << endl << endl;
2015 out << "cd $validateout;" << endl;
2016 out << "validateworkdir=`pwd`;" << endl << endl;
2017 out << "echo \"*******************************************************\"" << out_stream << endl;
2018 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2019 out << "" << endl;
2020 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2021 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2022 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2023 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2024 out << "ls -la ./" << out_stream << endl;
2025 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2026 out << "##################################################" << endl;
ebec370a 2027
2028 out << "" << endl;
2029 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2030 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2031 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2032 out << "" << endl;
2033
2034 out << "if [ ! -f stderr ] ; then" << endl;
2035 out << " error=1" << endl;
2036 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2037 out << " echo \"Error = $error\" " << out_stream << endl;
2038 out << "fi" << endl;
2039
2040 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2041 out << " error=1" << endl;
2042 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2043 out << " echo \"$parArch\" " << out_stream << endl;
2044 out << " echo \"Error = $error\" " << out_stream << endl;
2045 out << "fi" << endl;
2046
2047 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2048 out << " error=1" << endl;
2049 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2050 out << " echo \"$segViol\" " << out_stream << endl;
2051 out << " echo \"Error = $error\" " << out_stream << endl;
2052 out << "fi" << endl;
2053
2054 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2055 out << " error=1" << endl;
2056 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2057 out << " echo \"$segFault\" " << out_stream << endl;
2058 out << " echo \"Error = $error\" " << out_stream << endl;
2059 out << "fi" << endl;
2060
2061 // Part dedicated to the specific analyses running into the train
2062
c57f56b7 2063 TObjArray *arr = fOutputFiles.Tokenize(" ");
2064 TIter next1(arr);
2065 TString output_file;
2066 while ((os=(TObjString*)next1())) {
2067 output_file = os->GetString();
2068 Int_t index = output_file.Index("@");
2069 if (index > 0) output_file.Remove(index);
2070 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2071 out << " error=1" << endl;
2072 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2073 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2074 out << "fi" << endl;
2075 }
2076 delete arr;
2077 out << "if [ $error = 0 ] ; then" << endl;
2078 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2079 out << "fi" << endl;
2080
2081 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2082 out << "echo \"*******************************************************\"" << out_stream << endl;
2083 out << "cd -" << endl;
2084 out << "exit $error" << endl;
2085 }
2086 Bool_t copy = kTRUE;
2087 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2088 if (copy) {
2089 CdWork();
2090 TString workdir = gGrid->GetHomeDirectory();
2091 workdir += fGridWorkingDir;
0d5d317c 2092 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2093 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2094 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2095 }
2096}