]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Change TString additions
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
59 fArguments(),
60 fAnalysisMacro(),
61 fAnalysisSource(),
62 fAdditionalLibs(),
63 fSplitMode(),
64 fAPIVersion(),
65 fROOTVersion(),
66 fAliROOTVersion(),
67 fUser(),
68 fGridWorkingDir(),
69 fGridDataDir(),
70 fDataPattern(),
71 fGridOutputDir(),
72 fOutputArchive(),
73 fOutputFiles(),
74 fInputFormat(),
e7c71df0 75 fDatasetName(),
c57f56b7 76 fJDLName(),
bb885a9e 77 fMergeExcludes(),
f965131e 78 fIncludePath(),
bb885a9e 79 fCloseSE(),
0df6ccf2 80 fFriendChainName(),
c6cb3634 81 fJobTag(),
4e5c5506 82 fInputFiles(0),
83 fPackages(0)
c57f56b7 84{
85// Dummy ctor.
86 SetDefaults();
87}
88
89//______________________________________________________________________________
90AliAnalysisAlien::AliAnalysisAlien(const char *name)
91 :AliAnalysisGrid(name),
92 fGridJDL(NULL),
93 fPrice(0),
94 fTTL(0),
95 fSplitMaxInputFileNumber(0),
96 fMaxInitFailed(0),
97 fMasterResubmitThreshold(0),
bb885a9e 98 fNtestFiles(0),
319593fb 99 fNrunsPerMaster(0),
16a4353c 100 fMaxMergeFiles(0),
d2a409b2 101 fNsubmitted(0),
a3e84053 102 fProductionMode(0),
c57f56b7 103 fRunNumbers(),
104 fExecutable(),
105 fArguments(),
106 fAnalysisMacro(),
107 fAnalysisSource(),
108 fAdditionalLibs(),
109 fSplitMode(),
110 fAPIVersion(),
111 fROOTVersion(),
112 fAliROOTVersion(),
113 fUser(),
114 fGridWorkingDir(),
115 fGridDataDir(),
116 fDataPattern(),
117 fGridOutputDir(),
118 fOutputArchive(),
119 fOutputFiles(),
120 fInputFormat(),
e7c71df0 121 fDatasetName(),
c57f56b7 122 fJDLName(),
bb885a9e 123 fMergeExcludes(),
f965131e 124 fIncludePath(),
bb885a9e 125 fCloseSE(),
0df6ccf2 126 fFriendChainName(),
c6cb3634 127 fJobTag(),
4e5c5506 128 fInputFiles(0),
129 fPackages(0)
c57f56b7 130{
131// Default ctor.
132 SetDefaults();
133}
134
135//______________________________________________________________________________
136AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
137 :AliAnalysisGrid(other),
138 fGridJDL(NULL),
139 fPrice(other.fPrice),
140 fTTL(other.fTTL),
141 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
142 fMaxInitFailed(other.fMaxInitFailed),
143 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 144 fNtestFiles(other.fNtestFiles),
319593fb 145 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 146 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 147 fNsubmitted(other.fNsubmitted),
a3e84053 148 fProductionMode(other.fProductionMode),
c57f56b7 149 fRunNumbers(other.fRunNumbers),
150 fExecutable(other.fExecutable),
151 fArguments(other.fArguments),
152 fAnalysisMacro(other.fAnalysisMacro),
153 fAnalysisSource(other.fAnalysisSource),
154 fAdditionalLibs(other.fAdditionalLibs),
155 fSplitMode(other.fSplitMode),
156 fAPIVersion(other.fAPIVersion),
157 fROOTVersion(other.fROOTVersion),
158 fAliROOTVersion(other.fAliROOTVersion),
159 fUser(other.fUser),
160 fGridWorkingDir(other.fGridWorkingDir),
161 fGridDataDir(other.fGridDataDir),
162 fDataPattern(other.fDataPattern),
163 fGridOutputDir(other.fGridOutputDir),
164 fOutputArchive(other.fOutputArchive),
165 fOutputFiles(other.fOutputFiles),
166 fInputFormat(other.fInputFormat),
e7c71df0 167 fDatasetName(other.fDatasetName),
c57f56b7 168 fJDLName(other.fJDLName),
bb885a9e 169 fMergeExcludes(other.fMergeExcludes),
f965131e 170 fIncludePath(other.fIncludePath),
bb885a9e 171 fCloseSE(other.fCloseSE),
0df6ccf2 172 fFriendChainName(other.fFriendChainName),
c6cb3634 173 fJobTag(other.fJobTag),
4e5c5506 174 fInputFiles(0),
175 fPackages(0)
c57f56b7 176{
177// Copy ctor.
178 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 179 fRunRange[0] = other.fRunRange[0];
180 fRunRange[1] = other.fRunRange[1];
c57f56b7 181 if (other.fInputFiles) {
182 fInputFiles = new TObjArray();
183 TIter next(other.fInputFiles);
184 TObject *obj;
185 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
186 fInputFiles->SetOwner();
187 }
4e5c5506 188 if (other.fPackages) {
189 fPackages = new TObjArray();
190 TIter next(other.fPackages);
191 TObject *obj;
192 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
193 fPackages->SetOwner();
194 }
c57f56b7 195}
196
197//______________________________________________________________________________
198AliAnalysisAlien::~AliAnalysisAlien()
199{
200// Destructor.
201 if (fGridJDL) delete fGridJDL;
202 if (fInputFiles) delete fInputFiles;
4e5c5506 203 if (fPackages) delete fPackages;
c57f56b7 204}
205
206//______________________________________________________________________________
207AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
208{
209// Assignment.
210 if (this != &other) {
211 AliAnalysisGrid::operator=(other);
212 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
213 fPrice = other.fPrice;
214 fTTL = other.fTTL;
215 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
216 fMaxInitFailed = other.fMaxInitFailed;
217 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 218 fNtestFiles = other.fNtestFiles;
a3e84053 219 fNrunsPerMaster = other.fNrunsPerMaster;
220 fMaxMergeFiles = other.fMaxMergeFiles;
221 fNsubmitted = other.fNsubmitted;
222 fProductionMode = other.fProductionMode;
c57f56b7 223 fRunNumbers = other.fRunNumbers;
224 fExecutable = other.fExecutable;
225 fArguments = other.fArguments;
226 fAnalysisMacro = other.fAnalysisMacro;
227 fAnalysisSource = other.fAnalysisSource;
228 fAdditionalLibs = other.fAdditionalLibs;
229 fSplitMode = other.fSplitMode;
230 fAPIVersion = other.fAPIVersion;
231 fROOTVersion = other.fROOTVersion;
232 fAliROOTVersion = other.fAliROOTVersion;
233 fUser = other.fUser;
234 fGridWorkingDir = other.fGridWorkingDir;
235 fGridDataDir = other.fGridDataDir;
236 fDataPattern = other.fDataPattern;
237 fGridOutputDir = other.fGridOutputDir;
238 fOutputArchive = other.fOutputArchive;
239 fOutputFiles = other.fOutputFiles;
240 fInputFormat = other.fInputFormat;
e7c71df0 241 fDatasetName = other.fDatasetName;
c57f56b7 242 fJDLName = other.fJDLName;
bb885a9e 243 fMergeExcludes = other.fMergeExcludes;
f965131e 244 fIncludePath = other.fIncludePath;
bb885a9e 245 fCloseSE = other.fCloseSE;
0df6ccf2 246 fFriendChainName = other.fFriendChainName;
c6cb3634 247 fJobTag = other.fJobTag;
c57f56b7 248 if (other.fInputFiles) {
249 fInputFiles = new TObjArray();
250 TIter next(other.fInputFiles);
251 TObject *obj;
252 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
253 fInputFiles->SetOwner();
254 }
4e5c5506 255 if (other.fPackages) {
256 fPackages = new TObjArray();
257 TIter next(other.fPackages);
258 TObject *obj;
259 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
260 fPackages->SetOwner();
261 }
c57f56b7 262 }
263 return *this;
264}
265
f965131e 266//______________________________________________________________________________
267void AliAnalysisAlien::AddIncludePath(const char *path)
268{
269// Add include path in the remote analysis macro.
270 TString p(path);
271 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
272 else fIncludePath += Form("-I%s ", path);
273}
274
c57f56b7 275//______________________________________________________________________________
276void AliAnalysisAlien::AddRunNumber(Int_t run)
277{
278// Add a run number to the list of runs to be processed.
279 if (fRunNumbers.Length()) fRunNumbers += " ";
280 fRunNumbers += Form("%d", run);
281}
282
283//______________________________________________________________________________
284void AliAnalysisAlien::AddDataFile(const char *lfn)
285{
286// Adds a data file to the input to be analysed. The file should be a valid LFN
287// or point to an existing file in the alien workdir.
288 if (!fInputFiles) fInputFiles = new TObjArray();
289 fInputFiles->Add(new TObjString(lfn));
290}
291
292//______________________________________________________________________________
293Bool_t AliAnalysisAlien::Connect()
294{
295// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
296 if (gGrid && gGrid->IsConnected()) return kTRUE;
297 if (!gSystem->Getenv("alien_API_USER")) {
298 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
299 gSystem->Getenv("UID"));
300 return kFALSE;
301 }
302 if (!gGrid) {
303 Info("Connect", "Trying to connect to AliEn ...");
304 TGrid::Connect("alien://");
305 }
306 if (!gGrid || !gGrid->IsConnected()) {
307 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
308 return kFALSE;
309 }
310 fUser = gGrid->GetUser();
311 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
312 return kTRUE;
313}
314
315//______________________________________________________________________________
316void AliAnalysisAlien::CdWork()
317{
318// Check validity of alien workspace. Create directory if possible.
319 if (!Connect()) {
320 Error("CdWork", "Alien connection required");
321 return;
322 }
323 TString homedir = gGrid->GetHomeDirectory();
324 TString workdir = homedir + fGridWorkingDir;
325 if (!gGrid->Cd(workdir)) {
326 gGrid->Cd(homedir);
327 if (gGrid->Mkdir(workdir)) {
328 gGrid->Cd(fGridWorkingDir);
329 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
330 } else {
331 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
332 workdir.Data(), homedir.Data());
333 fGridWorkingDir = "";
334 }
335 }
336}
337
338//______________________________________________________________________________
339Bool_t AliAnalysisAlien::CheckInputData()
340{
341// Check validity of input data. If necessary, create xml files.
a8739e8a 342 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 343 if (!fGridDataDir.Length()) {
344 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
345 return kFALSE;
346 }
347 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
348 return kTRUE;
c57f56b7 349 }
350 // Process declared files
351 Bool_t is_collection = kFALSE;
352 Bool_t is_xml = kFALSE;
353 Bool_t use_tags = kFALSE;
354 Bool_t checked = kFALSE;
355 CdWork();
356 TString file;
357 TString workdir = gGrid->GetHomeDirectory();
358 workdir += fGridWorkingDir;
359 if (fInputFiles) {
360 TObjString *objstr;
361 TIter next(fInputFiles);
362 while ((objstr=(TObjString*)next())) {
363 file = workdir;
364 file += "/";
365 file += objstr->GetString();
366 // Store full lfn path
367 if (FileExists(file)) objstr->SetString(file);
368 else {
369 file = objstr->GetName();
370 if (!FileExists(objstr->GetName())) {
371 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
372 objstr->GetName(), workdir.Data());
373 return kFALSE;
374 }
375 }
376 Bool_t iscoll, isxml, usetags;
377 CheckDataType(file, iscoll, isxml, usetags);
378 if (!checked) {
379 checked = kTRUE;
380 is_collection = iscoll;
381 is_xml = isxml;
382 use_tags = usetags;
383 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
384 } else {
385 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
386 Error("CheckInputData", "Some conflict was found in the types of inputs");
387 return kFALSE;
388 }
389 }
390 }
391 }
392 // Process requested run numbers
a8739e8a 393 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 394 // Check validity of alien data directory
395 if (!fGridDataDir.Length()) {
396 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
397 return kFALSE;
398 }
399 if (!gGrid->Cd(fGridDataDir)) {
400 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
401 return kFALSE;
402 }
403 if (is_collection) {
404 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
405 return kFALSE;
406 }
407
408 if (checked && !is_xml) {
409 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
410 return kFALSE;
411 }
412 // Check validity of run number(s)
413 TObjArray *arr;
414 TObjString *os;
319593fb 415 Int_t nruns = 0;
416 TString schunk;
c57f56b7 417 TString path;
418 if (!checked) {
419 checked = kTRUE;
420 use_tags = fDataPattern.Contains("tag");
421 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
422 }
423 if (use_tags != fDataPattern.Contains("tag")) {
424 Error("CheckInputData", "Cannot mix input files using/not using tags");
425 return kFALSE;
426 }
427 if (fRunNumbers.Length()) {
a8739e8a 428 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 429 arr = fRunNumbers.Tokenize(" ");
430 TIter next(arr);
431 while ((os=(TObjString*)next())) {
432 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
433 if (!gGrid->Cd(path)) {
a8739e8a 434 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
435 continue;
c57f56b7 436 }
437 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
438 TString msg = "\n##### file: ";
439 msg += path;
440 msg += " type: xml_collection;";
441 if (use_tags) msg += " using_tags: Yes";
442 else msg += " using_tags: No";
443 Info("CheckDataType", msg.Data());
319593fb 444 if (fNrunsPerMaster<2) {
d2a409b2 445 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 446 } else {
447 nruns++;
448 if (((nruns-1)%fNrunsPerMaster) == 0) {
449 schunk = os->GetString();
450 }
451 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
452 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 453 AddDataFile(schunk);
319593fb 454 }
c57f56b7 455 }
456 delete arr;
a8739e8a 457 } else {
458 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
459 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
460 path = Form("%s/%d ", fGridDataDir.Data(), irun);
461 if (!gGrid->Cd(path)) {
462 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
463 continue;
464 }
465 path = Form("%s/%d.xml", workdir.Data(),irun);
466 TString msg = "\n##### file: ";
467 msg += path;
468 msg += " type: xml_collection;";
469 if (use_tags) msg += " using_tags: Yes";
470 else msg += " using_tags: No";
471 Info("CheckDataType", msg.Data());
319593fb 472 if (fNrunsPerMaster<2) {
d2a409b2 473 AddDataFile(Form("%d.xml",irun));
319593fb 474 } else {
475 nruns++;
476 if (((nruns-1)%fNrunsPerMaster) == 0) {
477 schunk = Form("%d", irun);
478 }
479 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
480 schunk += Form("_%d.xml", irun);
d2a409b2 481 AddDataFile(schunk);
319593fb 482 }
a8739e8a 483 }
c57f56b7 484 }
485 return kTRUE;
486}
487
488//______________________________________________________________________________
489Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
490{
491// Create dataset for the grid data directory + run number.
492 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
493 if (!Connect()) {
494 Error("CreateDataset", "Cannot create dataset with no grid connection");
495 return kFALSE;
496 }
497
498 // Cd workspace
499 CdWork();
500 TString workdir = gGrid->GetHomeDirectory();
501 workdir += fGridWorkingDir;
502
503 // Compose the 'find' command arguments
504 TString command;
505 TString options = "-x collection ";
bb885a9e 506 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 507 TString conditions = "";
508
509 TString file;
510 TString path;
319593fb 511 Int_t nruns = 0;
512 TString schunk;
ab254fd1 513 TGridCollection *cbase=0, *cadd=0;
d2a409b2 514 if (!fRunNumbers.Length() && !fRunRange[0]) {
515 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
516 // Make a single data collection from data directory.
517 path = fGridDataDir;
518 if (!gGrid->Cd(path)) {
519 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
520 return kFALSE;
521 }
522 CdWork();
523 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
524 else file = Form("%s.xml", gSystem->BaseName(path));
525 if (gSystem->AccessPathName(file)) {
526 command = "find ";
527 command += options;
528 command += path;
529 command += " ";
530 command += pattern;
531 command += conditions;
532 printf("command: %s\n", command.Data());
533 TGridResult *res = gGrid->Command(command);
534 if (res) delete res;
535 // Write standard output to file
536 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
537 }
538 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
539 // Copy xml file to alien space
540 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
541 if (!FileExists(file)) {
542 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
543 return kFALSE;
544 }
545 // Update list of files to be processed.
546 }
547 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
548 return kTRUE;
549 }
c57f56b7 550 // Several runs
a8739e8a 551 if (fRunNumbers.Length()) {
552 TObjArray *arr = fRunNumbers.Tokenize(" ");
553 TObjString *os;
554 TIter next(arr);
555 while ((os=(TObjString*)next())) {
556 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
673d4fb3 557 if (!gGrid->Cd(path)) continue;
95e5b448 558 CdWork();
a8739e8a 559 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
560 else file = Form("%s.xml", os->GetString().Data());
319593fb 561 // If local collection file does not exist, create it via 'find' command.
562 if (gSystem->AccessPathName(file)) {
563 command = "find ";
564 command += options;
565 command += path;
566 command += pattern;
567 command += conditions;
568 TGridResult *res = gGrid->Command(command);
569 if (res) delete res;
570 // Write standard output to file
571 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
572 }
a8739e8a 573 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 574 // Check if there is one run per master job.
575 if (fNrunsPerMaster<2) {
576 if (FileExists(file)) {
577 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
578 continue;
579 }
580 // Copy xml file to alien space
581 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
582 if (!FileExists(file)) {
583 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
584 delete arr;
585 return kFALSE;
586 }
587 } else {
588 nruns++;
589 if (((nruns-1)%fNrunsPerMaster) == 0) {
590 schunk = os->GetString();
591 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
592 } else {
593 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
594 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
595 cbase->Add(cadd);
596 delete cadd;
597 }
598 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
599 continue;
600 }
601 schunk += Form("_%s.xml", os->GetString().Data());
602 if (FileExists(schunk)) {
603 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
604 continue;
605 }
606 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
607 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
608 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
609 if (!FileExists(schunk)) {
610 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
611 delete arr;
612 return kFALSE;
613 }
614 }
a8739e8a 615 }
616 delete arr;
617 } else {
618 // Process a full run range.
619 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
620 path = Form("%s/%d ", fGridDataDir.Data(), irun);
673d4fb3 621 if (!gGrid->Cd(path)) continue;
95e5b448 622 CdWork();
a8739e8a 623 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
624 else file = Form("%d.xml", irun);
319593fb 625 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 626 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
627// gGrid->Rm(file);
628 continue;
629 }
319593fb 630 // If local collection file does not exist, create it via 'find' command.
631 if (gSystem->AccessPathName(file)) {
632 command = "find ";
633 command += options;
634 command += path;
635 command += pattern;
636 command += conditions;
637 TGridResult *res = gGrid->Command(command);
638 if (res) delete res;
639 // Write standard output to file
640 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
641 }
a8739e8a 642 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 643 // Check if there is one run per master job.
644 if (fNrunsPerMaster<2) {
645 if (FileExists(file)) {
646 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
647 continue;
648 }
649 // Copy xml file to alien space
650 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
651 if (!FileExists(file)) {
652 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
653 return kFALSE;
654 }
655 } else {
656 nruns++;
95e5b448 657 // Check if the collection for the chunk exist locally.
658 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
659 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
319593fb 660 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
661 if (((nruns-1)%fNrunsPerMaster) == 0) {
662 schunk = Form("%d", irun);
663 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
664 } else {
665 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
666 cbase->Add(cadd);
667 delete cadd;
668 }
669 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
670 continue;
671 }
672 schunk += Form("_%d.xml", irun);
673 if (FileExists(schunk)) {
674 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
675 continue;
676 }
677 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
678 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 679 if (FileExists(schunk)) {
680 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
681 continue;
682 }
319593fb 683 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
684 if (!FileExists(schunk)) {
685 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
686 return kFALSE;
687 }
688 }
c57f56b7 689 }
a8739e8a 690 }
c57f56b7 691 return kTRUE;
692}
693
694//______________________________________________________________________________
695Bool_t AliAnalysisAlien::CreateJDL()
696{
697// Generate a JDL file according to current settings. The name of the file is
698// specified by fJDLName.
699 Bool_t error = kFALSE;
700 TObjArray *arr = 0;
701 Bool_t copy = kTRUE;
702 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
703 Bool_t generate = kTRUE;
704 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
705 if (!Connect()) {
706 Error("CreateJDL", "Alien connection required");
707 return kFALSE;
708 }
709 // Check validity of alien workspace
710 CdWork();
711 TString workdir = gGrid->GetHomeDirectory();
712 workdir += fGridWorkingDir;
713 if (generate) {
714 TObjString *os;
715 if (!fInputFiles) {
716 Error("CreateJDL()", "Define some input files for your analysis.");
717 error = kTRUE;
718 }
719 // Compose list of input files
720 // Check if output files were defined
721 if (!fOutputFiles.Length()) {
722 Error("CreateJDL", "You must define at least one output file");
723 error = kTRUE;
724 }
725 // Check if an output directory was defined and valid
726 if (!fGridOutputDir.Length()) {
727 Error("CreateJDL", "You must define AliEn output directory");
728 error = kTRUE;
729 } else {
d2a409b2 730 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
c57f56b7 731 if (!gGrid->Cd(fGridOutputDir)) {
732 if (gGrid->Mkdir(fGridOutputDir)) {
733 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
734 } else {
735 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 736 // error = kTRUE;
c57f56b7 737 }
738 }
739 gGrid->Cd(workdir);
740 }
741 // Exit if any error up to now
742 if (error) return kFALSE;
743 // Set JDL fields
744 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 745 fGridJDL->SetExecutable(fExecutable);
c57f56b7 746// fGridJDL->SetTTL((UInt_t)fTTL);
747 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
748 if (fMaxInitFailed > 0)
749 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
750 if (fSplitMaxInputFileNumber > 0)
751 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
752 if (fSplitMode.Length())
753 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
754// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
755 if (fAliROOTVersion.Length())
756 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
757 if (fROOTVersion.Length())
758 fGridJDL->AddToPackages("ROOT", fROOTVersion);
759 if (fAPIVersion.Length())
760 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
761 fGridJDL->SetInputDataListFormat(fInputFormat);
762 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 763 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
764 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
765 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
766 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
767 if (fAdditionalLibs.Length()) {
768 arr = fAdditionalLibs.Tokenize(" ");
769 TIter next(arr);
770 while ((os=(TObjString*)next())) {
771 if (os->GetString().Contains(".so")) continue;
772 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
773 }
774 delete arr;
775 }
4e5c5506 776 if (fPackages) {
777 TIter next(fPackages);
778 TObject *obj;
779 while ((obj=next()))
780 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
781 }
c57f56b7 782 if (fOutputArchive.Length()) {
783 arr = fOutputArchive.Tokenize(" ");
784 TIter next(arr);
785 while ((os=(TObjString*)next()))
43da816a 786 if (!os->GetString().Contains("@") && fCloseSE.Length())
787 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
788 else
789 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 790 delete arr;
791 }
c57f56b7 792 arr = fOutputFiles.Tokenize(" ");
793 TIter next(arr);
43da816a 794 while ((os=(TObjString*)next())) {
e1eaf596 795 // Ignore ouputs in jdl that are also in outputarchive
796 TString sout = os->GetString();
797 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
798 if (fOutputArchive.Contains(sout)) continue;
43da816a 799 if (!os->GetString().Contains("@") && fCloseSE.Length())
800 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
801 else
802 fGridJDL->AddToOutputSandbox(os->GetString());
803 }
c57f56b7 804 delete arr;
805// fGridJDL->SetPrice((UInt_t)fPrice);
806 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
807 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
808 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 809 // Write a jdl with 2 input parameters: collection name and output dir name.
810 WriteJDL(copy);
c57f56b7 811 }
812 // Copy jdl to grid workspace
a8739e8a 813 if (copy) {
c57f56b7 814 if (fAdditionalLibs.Length()) {
e7c71df0 815 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 816 TObjString *os;
817 TIter next(arr);
818 while ((os=(TObjString*)next())) {
c57f56b7 819 if (os->GetString().Contains(".so")) continue;
4e5c5506 820 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 821 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
822 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
823 }
824 delete arr;
825 }
4e5c5506 826 if (fPackages) {
827 TIter next(fPackages);
828 TObject *obj;
829 while ((obj=next())) {
830 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
831 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
832 }
833 }
c57f56b7 834 }
835 return kTRUE;
836}
837
a8739e8a 838//______________________________________________________________________________
d2a409b2 839Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 840{
841// Writes one or more JDL's corresponding to findex. If findex is negative,
842// all run numbers are considered in one go (jdl). For non-negative indices
843// they correspond to the indices in the array fInputFiles.
844 if (!fInputFiles) return kFALSE;
845 TObjString *os;
a8739e8a 846 TString workdir = gGrid->GetHomeDirectory();
847 workdir += fGridWorkingDir;
d2a409b2 848
849 if (!fRunNumbers.Length() && !fRunRange[0]) {
850 // One jdl with no parameters in case input data is specified by name.
a8739e8a 851 TIter next(fInputFiles);
852 while ((os=(TObjString*)next()))
853 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 854 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 855 } else {
d2a409b2 856 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
857 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 858 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 859 }
860
861
862 // Generate the JDL as a string
863 TString sjdl = fGridJDL->Generate();
864 Int_t index;
865 index = sjdl.Index("Executable");
866 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
867 index = sjdl.Index("Split ");
868 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
869 index = sjdl.Index("SplitMaxInputFileNumber");
870 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
871 index = sjdl.Index("InputDataCollection");
872 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 873 index = sjdl.Index("InputFile");
874 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
875 index = sjdl.Index("InputDataList ");
876 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
877 index = sjdl.Index("InputDataListFormat");
878 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
879 index = sjdl.Index("Price");
880 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
881 index = sjdl.Index("Requirements");
882 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
883 index = sjdl.Index("Packages");
884 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
885 index = sjdl.Index("User =");
886 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
887 index = sjdl.Index("TTL");
888 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
889 index = sjdl.Index("OutputFile");
890 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
891 index = sjdl.Index("OutputDir");
892 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
893 index = sjdl.Index("OutputArchive");
894 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
895 index = sjdl.Index("MaxInitFailed");
896 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
897 index = sjdl.Index("MasterResubmitThreshold");
898 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
899 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
900 index = sjdl.Index("Validationcommand");
901 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
902 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
903 sjdl.ReplaceAll("(member", "\n (member");
904 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
905 sjdl.ReplaceAll("{", "{\n ");
906 sjdl.ReplaceAll("};", "\n};");
907 sjdl.ReplaceAll("{\n \n", "{\n");
908 sjdl.ReplaceAll("\n\n", "\n");
909 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
910 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 911 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 912 index = sjdl.Index("JDLVariables");
913 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
914 // Write jdl to file
a8739e8a 915 ofstream out;
d2a409b2 916 out.open(fJDLName.Data(), ios::out);
a8739e8a 917 if (out.bad()) {
d2a409b2 918 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 919 return kFALSE;
920 }
921 out << sjdl << endl;
922
923 // Copy jdl to grid workspace
924 if (!copy) {
925 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
926 } else {
d2a409b2 927 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
928 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 929 if (fProductionMode)
b5fe9cba 930 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 931 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 932 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 933 }
934 return kTRUE;
935}
936
c57f56b7 937//______________________________________________________________________________
5513444a 938Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 939{
940// Returns true if file exists.
5513444a 941 if (!gGrid) return kFALSE;
c57f56b7 942 TGridResult *res = gGrid->Ls(lfn);
943 if (!res) return kFALSE;
944 TMap *map = dynamic_cast<TMap*>(res->At(0));
945 if (!map) {
946 delete res;
947 return kFALSE;
948 }
949 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
950 if (!objs || !objs->GetString().Length()) {
951 delete res;
952 return kFALSE;
953 }
954 delete res;
955 return kTRUE;
956}
957
958//______________________________________________________________________________
959void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
960{
961// Check input data type.
962 is_collection = kFALSE;
963 is_xml = kFALSE;
964 use_tags = kFALSE;
965 if (!gGrid) {
966 Error("CheckDataType", "No connection to grid");
967 return;
968 }
969 is_collection = IsCollection(lfn);
970 TString msg = "\n##### file: ";
971 msg += lfn;
972 if (is_collection) {
973 msg += " type: raw_collection;";
974 // special treatment for collections
975 is_xml = kFALSE;
976 // check for tag files in the collection
977 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
978 if (!res) {
979 msg += " using_tags: No (unknown)";
980 Info("CheckDataType", msg.Data());
981 return;
982 }
983 const char* typeStr = res->GetKey(0, "origLFN");
984 if (!typeStr || !strlen(typeStr)) {
985 msg += " using_tags: No (unknown)";
986 Info("CheckDataType", msg.Data());
987 return;
988 }
989 TString file = typeStr;
990 use_tags = file.Contains(".tag");
991 if (use_tags) msg += " using_tags: Yes";
992 else msg += " using_tags: No";
993 Info("CheckDataType", msg.Data());
994 return;
995 }
996 TString slfn(lfn);
997 slfn.ToLower();
998 is_xml = slfn.Contains(".xml");
999 if (is_xml) {
1000 // Open xml collection and check if there are tag files inside
1001 msg += " type: xml_collection;";
1002 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1003 if (!coll) {
1004 msg += " using_tags: No (unknown)";
1005 Info("CheckDataType", msg.Data());
1006 return;
1007 }
1008 TMap *map = coll->Next();
1009 if (!map) {
1010 msg += " using_tags: No (unknown)";
1011 Info("CheckDataType", msg.Data());
1012 return;
1013 }
1014 map = (TMap*)map->GetValue("");
1015 TString file;
1016 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1017 use_tags = file.Contains(".tag");
1018 delete coll;
1019 if (use_tags) msg += " using_tags: Yes";
1020 else msg += " using_tags: No";
1021 Info("CheckDataType", msg.Data());
1022 return;
1023 }
1024 use_tags = slfn.Contains(".tag");
1025 if (slfn.Contains(".root")) msg += " type: root file;";
1026 else msg += " type: unhnown file;";
1027 if (use_tags) msg += " using_tags: Yes";
1028 else msg += " using_tags: No";
1029 Info("CheckDataType", msg.Data());
1030}
1031
4e5c5506 1032//______________________________________________________________________________
1033void AliAnalysisAlien::EnablePackage(const char *package)
1034{
1035// Enables a par file supposed to exist in the current directory.
1036 TString pkg(package);
1037 pkg.ReplaceAll(".par", "");
1038 pkg += ".par";
1039 if (gSystem->AccessPathName(pkg)) {
1040 Error("EnablePackage", "Package %s not found", pkg.Data());
1041 return;
1042 }
1043 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1044 Info("EnablePackage", "AliEn plugin will use .par packages");
1045 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1046 if (!fPackages) {
1047 fPackages = new TObjArray();
1048 fPackages->SetOwner();
1049 }
1050 fPackages->Add(new TObjString(pkg));
1051}
1052
d2a409b2 1053//______________________________________________________________________________
1054const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1055{
1056// Get job status for all jobs with jobid>jobidstart.
1057 static char mstatus[20];
1058 mstatus[0] = '\0';
1059 nrunning = 0;
1060 nwaiting = 0;
1061 nerror = 0;
1062 ndone = 0;
1063 TGridJobStatusList *list = gGrid->Ps("");
1064 if (!list) return mstatus;
1065 Int_t nentries = list->GetSize();
1066 TGridJobStatus *status;
1067 Int_t pid;
1068 for (Int_t ijob=0; ijob<nentries; ijob++) {
1069 status = (TGridJobStatus *)list->At(ijob);
1070 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1071 if (pid<jobidstart) continue;
1072 if (pid == lastid) {
1073 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1074 }
1075 switch (status->GetStatus()) {
1076 case TGridJobStatus::kWAITING:
1077 nwaiting++; break;
1078 case TGridJobStatus::kRUNNING:
1079 nrunning++; break;
1080 case TGridJobStatus::kABORTED:
1081 case TGridJobStatus::kFAIL:
1082 case TGridJobStatus::kUNKNOWN:
1083 nerror++; break;
1084 case TGridJobStatus::kDONE:
1085 ndone++;
1086 }
1087 }
1088 list->Delete();
1089 delete list;
1090 return mstatus;
1091}
1092
c57f56b7 1093//______________________________________________________________________________
1094Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1095{
1096// Returns true if file is a collection. Functionality duplicated from
1097// TAlien::Type() because we don't want to directly depend on TAlien.
1098 if (!gGrid) {
1099 Error("IsCollection", "No connection to grid");
1100 return kFALSE;
1101 }
1102 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1103 if (!res) return kFALSE;
1104 const char* typeStr = res->GetKey(0, "type");
1105 if (!typeStr || !strlen(typeStr)) return kFALSE;
1106 if (!strcmp(typeStr, "collection")) return kTRUE;
1107 delete res;
1108 return kFALSE;
1109}
1110
16a4353c 1111//______________________________________________________________________________
1112void AliAnalysisAlien::Print(Option_t *) const
1113{
1114// Print current plugin settings.
1115 printf("### AliEn analysis plugin current settings ###\n");
a3e84053 1116 printf("= Production mode:______________________________ %d\n", fProductionMode);
16a4353c 1117 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1118 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1119 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1120 if (fUser.Length())
1121 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1122 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1123 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1124 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1125 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1126 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1127 if (fRunNumbers.Length())
1128 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1129 if (fRunRange[0])
1130 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1131 if (!fRunRange[0] && !fRunNumbers.Length()) {
1132 TIter next(fInputFiles);
1133 TObject *obj;
1134 TString list;
1135 while ((obj=next())) list += obj->GetName();
1136 printf("= Input files to be processed: _________________ %s\n", list.Data());
1137 }
1138 if (TestBit(AliAnalysisGrid::kTest))
1139 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1140 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1141 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1142 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1143 printf("=====================================================================\n");
1144 printf("= Job price: ___________________________________ %d\n", fPrice);
1145 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1146 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1147 if (fMaxInitFailed>0)
1148 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1149 if (fMasterResubmitThreshold>0)
1150 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1151 if (fNrunsPerMaster>0)
1152 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
16a4353c 1153 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1154 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
1155 if (fArguments.Length())
1156 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1157 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1158 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1159 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1160 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1161 if (fDatasetName)
1162 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1163 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1164 if (fIncludePath.Data())
1165 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1166 if (fCloseSE.Length())
1167 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1168 if (fFriendChainName.Length())
1169 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1170 if (fPackages) {
1171 TIter next(fPackages);
1172 TObject *obj;
1173 TString list;
1174 while ((obj=next())) list += obj->GetName();
1175 printf("= Par files to be used: ________________________ %s\n", list.Data());
1176 }
1177}
1178
c57f56b7 1179//______________________________________________________________________________
1180void AliAnalysisAlien::SetDefaults()
1181{
1182// Set default values for everything. What cannot be filled will be left empty.
1183 if (fGridJDL) delete fGridJDL;
1184 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1185 fPrice = 1;
1186 fTTL = 30000;
1187 fSplitMaxInputFileNumber = 100;
1188 fMaxInitFailed = 0;
1189 fMasterResubmitThreshold = 0;
bb885a9e 1190 fNtestFiles = 10;
a8739e8a 1191 fRunRange[0] = 0;
1192 fRunRange[1] = 0;
319593fb 1193 fNrunsPerMaster = 1;
16a4353c 1194 fMaxMergeFiles = 100;
c57f56b7 1195 fRunNumbers = "";
1196 fExecutable = "analysis.sh";
1197 fArguments = "";
1198 fAnalysisMacro = "myAnalysis.C";
1199 fAnalysisSource = "";
1200 fAdditionalLibs = "";
1201 fSplitMode = "se";
1202 fAPIVersion = "";
1203 fROOTVersion = "";
1204 fAliROOTVersion = "";
1205 fUser = ""; // Your alien user name
1206 fGridWorkingDir = "";
1207 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1208 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1209 fFriendChainName = "";
c57f56b7 1210 fGridOutputDir = "output";
1211 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1212 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1213 fInputFormat = "xml-single";
1214 fJDLName = "analysis.jdl";
c6cb3634 1215 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1216 fMergeExcludes = "";
c57f56b7 1217}
1218
1219//______________________________________________________________________________
1220Bool_t AliAnalysisAlien::MergeOutputs()
1221{
1222// Merge analysis outputs existing in the AliEn space.
1223 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1224 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1225 if (!Connect()) {
1226 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1227 return kFALSE;
1228 }
1229 // Get the output path
d2a409b2 1230 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1231 if (!gGrid->Cd(fGridOutputDir)) {
c57f56b7 1232 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1233 return kFALSE;
1234 }
1235 if (!fOutputFiles.Length()) {
1236 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1237 return kFALSE;
1238 }
1239 TObjArray *list = fOutputFiles.Tokenize(" ");
1240 TIter next(list);
1241 TObjString *str;
1242 TString command;
1243 TString output_file;
16a4353c 1244 TString output_chunk;
1245 TString previous_chunk;
1246 Int_t count_chunk = 0;
1247 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1248 Bool_t merged = kTRUE;
1249 while((str=(TObjString*)next())) {
1250 output_file = str->GetString();
1251 Int_t index = output_file.Index("@");
1252 if (index > 0) output_file.Remove(index);
319593fb 1253 // Skip already merged outputs
1254 if (!gSystem->AccessPathName(output_file)) {
1255 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1256 continue;
1257 }
bb885a9e 1258 if (fMergeExcludes.Length() &&
1259 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1260 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1261 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
c57f56b7 1262 printf("command: %s\n", command.Data());
1263 TGridResult *res = gGrid->Command(command);
1264 if (!res) continue;
1265 TFileMerger *fm = 0;
1266 TIter nextmap(res);
ab254fd1 1267 TMap *map = 0;
16a4353c 1268 previous_chunk = "";
1269 count_chunk = 0;
319593fb 1270 // Check if there is a merge operation to resume
1271 output_chunk = output_file;
1272 output_chunk.ReplaceAll(".root", "_*.root");
1273 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1274 while (1) {
1275 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1276 if (!map) {
1277 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1278 delete res;
1279 return kFALSE;
1280 }
1281 output_chunk = output_file;
1282 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1283 printf("%s\n", output_chunk.Data());
1284 count_chunk++;
1285 if (gSystem->AccessPathName(output_chunk)) continue;
1286 // Merged file with chunks up to <count_chunk> found
1287 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1288 previous_chunk = output_chunk;
1289 break;
1290 }
1291 }
16a4353c 1292 count_zero = fMaxMergeFiles;
c57f56b7 1293 while ((map=(TMap*)nextmap())) {
16a4353c 1294 // Loop 'find' results and get next LFN
1295 if (count_zero == fMaxMergeFiles) {
1296 // First file in chunk - create file merger and add previous chunk if any.
1297 fm = new TFileMerger(kFALSE);
1298 fm->SetFastMethod(kTRUE);
1299 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1300 output_chunk = output_file;
1301 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1302 }
1303 // If last file found, put merged results in the output file
1304 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1305 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1306 if (!objs || !objs->GetString().Length()) {
16a4353c 1307 // Nothing found - skip this output
c57f56b7 1308 delete res;
16a4353c 1309 delete fm;
1310 break;
1311 }
1312 // Add file to be merged and decrement chunk counter.
1313 fm->AddFile(objs->GetString());
1314 count_zero--;
1315 if (count_zero==0 || map == res->Last()) {
1316 fm->OutputFile(output_chunk);
1317 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1318 // Nothing found - skip this output
1319 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1320 delete res;
1321 delete fm;
1322 break;
1323 }
1324 // Merge the outputs, then go to next chunk
1325 if (!fm->Merge()) {
1326 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1327 delete res;
1328 delete fm;
1329 merged = kFALSE;
1330 break;
1331 } else {
1332 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1333 gSystem->Unlink(previous_chunk);
16a4353c 1334 }
1335 if (map == res->Last()) {
1336 delete res;
1337 delete fm;
1338 break;
1339 }
1340 count_chunk++;
1341 count_zero = fMaxMergeFiles;
1342 previous_chunk = output_chunk;
c57f56b7 1343 }
c57f56b7 1344 }
c57f56b7 1345 }
1346 if (!merged) {
1347 Error("MergeOutputs", "Terminate() will NOT be executed");
1348 }
1349 return merged;
1350}
1351
bb885a9e 1352//______________________________________________________________________________
1353void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1354{
1355// Use the output files connected to output containers from the analysis manager
1356// rather than the files defined by SetOutputFiles
1357 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1358 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1359 analysis manager");
1360 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1361}
1362
5513444a 1363//______________________________________________________________________________
1364Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1365{
1366// Start remote grid analysis.
1367
43da816a 1368 // Check if output files have to be taken from the analysis manager
1369 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1370 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1371 if (!mgr || !mgr->IsInitialized()) {
1372 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1373 return kFALSE;
43da816a 1374 }
1375 fOutputFiles = "";
1376 TIter next(mgr->GetOutputs());
1377 AliAnalysisDataContainer *output;
1378 while ((output=(AliAnalysisDataContainer*)next())) {
1379 const char *filename = output->GetFileName();
1380 if (!(strcmp(filename, "default"))) {
1381 if (!mgr->GetOutputEventHandler()) continue;
1382 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1383 }
1384 if (fOutputFiles.Length()) fOutputFiles += " ";
1385 fOutputFiles += filename;
1386 }
c07b9ce2 1387 // Add extra files registered to the analysis manager
1388 if (mgr->GetExtraFiles().Length()) {
1389 if (fOutputFiles.Length()) fOutputFiles += " ";
1390 fOutputFiles += mgr->GetExtraFiles();
1391 }
43da816a 1392 }
f7b1cbc2 1393// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1394 if (TestBit(AliAnalysisGrid::kOffline)) {
1395 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1396 \n there nor any job run. You can revise the JDL and analysis \
1397 \n macro then run the same in \"submit\" mode.");
1398 } else if (TestBit(AliAnalysisGrid::kTest)) {
1399 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1400 \n dataset.");
1401 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1402 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1403 \n space and job submitted.");
1404 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1405 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1406 return kTRUE;
c57f56b7 1407 } else {
1408 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1409 }
1410
1411 if (!Connect()) {
1412 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1413 return kFALSE;
16a4353c 1414 }
1415 Print();
c57f56b7 1416 if (!CheckInputData()) {
1417 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1418 return kFALSE;
c57f56b7 1419 }
1420 CreateDataset(fDataPattern);
1421 WriteAnalysisFile();
1422 WriteAnalysisMacro();
1423 WriteExecutable();
1424 WriteValidationScript();
5513444a 1425 if (!CreateJDL()) return kFALSE;
1426 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1427 if (TestBit(AliAnalysisGrid::kTest)) {
1428 // Locally testing the analysis
1429 Info("StartAnalysis", "\n_______________________________________________________________________ \
1430 \n Running analysis script in a daughter shell as on a worker node \
1431 \n_______________________________________________________________________");
1432 TObjArray *list = fOutputFiles.Tokenize(" ");
1433 TIter next(list);
1434 TObjString *str;
1435 TString output_file;
1436 while((str=(TObjString*)next())) {
1437 output_file = str->GetString();
1438 Int_t index = output_file.Index("@");
1439 if (index > 0) output_file.Remove(index);
43da816a 1440 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1441 }
1442 delete list;
1443 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1444 gSystem->Exec("bash validate.sh");
1445// gSystem->Exec("cat stdout");
5513444a 1446 return kFALSE;
c57f56b7 1447 }
5513444a 1448 // Check if submitting is managed by LPM manager
a3e84053 1449 if (fProductionMode) {
5513444a 1450 TString prodfile = fJDLName;
1451 prodfile.ReplaceAll(".jdl", ".prod");
1452 WriteProductionFile(prodfile);
1453 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1454 return kFALSE;
1455 }
a8739e8a 1456 // Submit AliEn job(s)
d2a409b2 1457 gGrid->Cd(fGridOutputDir);
a8739e8a 1458 TGridResult *res;
c57f56b7 1459 TString jobID = "";
d2a409b2 1460 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1461 // Submit a given xml or a set of runs
a8739e8a 1462 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1463 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1464 if (res) {
1465 const char *cjobId = res->GetKey(0,"jobId");
1466 if (!cjobId) {
1467 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1468 return kFALSE;
a8739e8a 1469 } else {
1470 Info("StartAnalysis", "\n_______________________________________________________________________ \
1471 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1472 \n_______________________________________________________________________",
1473 fJDLName.Data(), cjobId);
1474 jobID = cjobId;
1475 }
1476 delete res;
1477 }
1478 } else {
d2a409b2 1479 // Submit for a range of enumeration of runs.
1480 Submit();
c57f56b7 1481 }
a8739e8a 1482
c57f56b7 1483 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1484 \n You may exit at any time and terminate the job later using the option <terminate> \
1485 \n ##################################################################################", jobID.Data());
bb885a9e 1486 gSystem->Exec("aliensh");
5513444a 1487 return kTRUE;
c57f56b7 1488}
1489
d2a409b2 1490//______________________________________________________________________________
1491void AliAnalysisAlien::Submit()
1492{
1493// Submit all master jobs.
1494 Int_t nmasterjobs = fInputFiles->GetEntries();
1495 Long_t tshoot = gSystem->Now();
1496 if (!fNsubmitted) SubmitNext();
1497 while (fNsubmitted < nmasterjobs) {
1498 Long_t now = gSystem->Now();
1499 if ((now-tshoot)>30000) {
1500 tshoot = now;
1501 SubmitNext();
1502 }
1503 }
1504}
1505
1506//______________________________________________________________________________
1507void AliAnalysisAlien::SubmitNext()
1508{
1509// Submit next bunch of master jobs if the queue is free.
1510 static Bool_t iscalled = kFALSE;
1511 static Int_t firstmaster = 0;
1512 static Int_t lastmaster = 0;
1513 static Int_t npermaster = 0;
1514 if (iscalled) return;
1515 iscalled = kTRUE;
1516 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1517 Int_t ntosubmit = 0;
1518 TGridResult *res;
1519 TString jobID = "";
1520 if (!fNsubmitted) ntosubmit = 1;
1521 else {
1522 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
1523 printf("=== master %d: %s\n", lastmaster, status.Data());
1524 // If last master not split, just return
1525 if (status != "SPLIT") {iscalled = kFALSE; return;}
1526 // No more than 100 waiting jobs
1527 if (nwaiting>100) {iscalled = kFALSE; return;}
1528 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1529 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
1530 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
1531 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1532 }
1533 Int_t nmasterjobs = fInputFiles->GetEntries();
1534 for (Int_t i=0; i<ntosubmit; i++) {
1535 // Submit for a range of enumeration of runs.
1536 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1537 TString query;
1538 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
1539 printf("********* %s\n",query.Data());
1540 res = gGrid->Command(query);
1541 if (res) {
98ca124f 1542 TString cjobId1 = res->GetKey(0,"jobId");
1543 if (!cjobId1.Length()) {
d2a409b2 1544 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1545 iscalled = kFALSE;
1546 return;
1547 } else {
1548 Info("StartAnalysis", "\n_______________________________________________________________________ \
1549 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1550 \n_______________________________________________________________________",
98ca124f 1551 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1552 jobID += cjobId1;
1553 jobID += " ";
98ca124f 1554 lastmaster = cjobId1.Atoi();
d2a409b2 1555 if (!firstmaster) firstmaster = lastmaster;
1556 fNsubmitted++;
1557 }
1558 delete res;
1559 }
1560 }
1561 iscalled = kFALSE;
1562}
1563
c57f56b7 1564//______________________________________________________________________________
1565void AliAnalysisAlien::WriteAnalysisFile()
1566{
1567// Write current analysis manager into the file analysis.root
1568 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1569 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1570 if (!mgr || !mgr->IsInitialized()) {
1571 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1572 return;
1573 }
1574 // Check analysis type
1575 TObject *handler;
1576 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1577 handler = (TObject*)mgr->GetInputEventHandler();
1578 if (handler) {
1579 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1580 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1581 }
1582 TDirectory *cdir = gDirectory;
1583 TFile *file = TFile::Open("analysis.root", "RECREATE");
1584 if (file) {
1585 mgr->Write();
1586 delete file;
1587 }
1588 if (cdir) cdir->cd();
1589 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1590 }
1591 Bool_t copy = kTRUE;
1592 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1593 if (copy) {
1594 CdWork();
1595 TString workdir = gGrid->GetHomeDirectory();
1596 workdir += fGridWorkingDir;
1597 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1598 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1599 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1600 }
1601}
1602
1603//______________________________________________________________________________
1604void AliAnalysisAlien::WriteAnalysisMacro()
1605{
1606// Write the analysis macro that will steer the analysis in grid mode.
1607 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1608 ofstream out;
1609 out.open(fAnalysisMacro.Data(), ios::out);
1610 if (!out.good()) {
1611 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1612 return;
1613 }
1614 TString func = fAnalysisMacro;
1615 TString type = "ESD";
1616 TString comment = "// Analysis using ";
1617 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1618 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1619 type = "AOD";
1620 comment += "AOD";
1621 }
0df6ccf2 1622 if (type!="AOD" && fFriendChainName!="") {
1623 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1624 return;
1625 }
c57f56b7 1626 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1627 else comment += " data";
1628 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1629 func.ReplaceAll(".C", "");
1630 out << "void " << func.Data() << "()" << endl;
1631 out << "{" << endl;
1632 out << comment.Data() << endl;
1633 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1634 out << " TStopwatch timer;" << endl;
1635 out << " timer.Start();" << endl << endl;
c57f56b7 1636 out << "// load base root libraries" << endl;
1637 out << " gSystem->Load(\"libTree\");" << endl;
1638 out << " gSystem->Load(\"libGeom\");" << endl;
1639 out << " gSystem->Load(\"libVMC\");" << endl;
1640 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1641 out << "// Load analysis framework libraries" << endl;
4e5c5506 1642 if (!fPackages) {
4e5c5506 1643 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1644 out << " gSystem->Load(\"libESD\");" << endl;
1645 out << " gSystem->Load(\"libAOD\");" << endl;
1646 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1647 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1648 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1649 } else {
4e5c5506 1650 TIter next(fPackages);
1651 TObject *obj;
57377eb5 1652 TString pkgname;
1653 Bool_t hasSTEERBase = kFALSE;
1654 Bool_t hasESD = kFALSE;
1655 Bool_t hasAOD = kFALSE;
1656 Bool_t hasANALYSIS = kFALSE;
1657 Bool_t hasANALYSISalice = kFALSE;
1658 Bool_t hasCORRFW = kFALSE;
1659 while ((obj=next())) {
1660 pkgname = obj->GetName();
4478e6f1 1661 if (pkgname == "STEERBase" ||
1662 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1663 if (pkgname == "ESD" ||
1664 pkgname == "ESD.par") hasESD = kTRUE;
1665 if (pkgname == "AOD" ||
1666 pkgname == "AOD.par") hasAOD = kTRUE;
1667 if (pkgname == "ANALYSIS" ||
1668 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1669 if (pkgname == "ANALYSISalice" ||
1670 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1671 if (pkgname == "CORRFW" ||
1672 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1673 }
1674 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1675 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1676 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1677 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1678 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1679 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1680 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1681 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1682 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1683 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1684 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1685 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1686 out << "// Compile other par packages" << endl;
1687 next.Reset();
fcc9bb6f 1688 while ((obj=next())) {
1689 pkgname = obj->GetName();
4478e6f1 1690 if (pkgname == "STEERBase" ||
1691 pkgname == "STEERBase.par" ||
1692 pkgname == "ESD" ||
1693 pkgname == "ESD.par" ||
1694 pkgname == "AOD" ||
1695 pkgname == "AOD.par" ||
1696 pkgname == "ANALYSIS" ||
1697 pkgname == "ANALYSIS.par" ||
1698 pkgname == "ANALYSISalice" ||
1699 pkgname == "ANALYSISalice.par" ||
1700 pkgname == "CORRFW" ||
1701 pkgname == "CORRFW.par") continue;
4e5c5506 1702 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1703 }
4e5c5506 1704 }
43da816a 1705 out << "// include path" << endl;
1706 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1707 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1708 if (fAdditionalLibs.Length()) {
1709 out << "// Add aditional AliRoot libraries" << endl;
1710 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1711 TIter next(list);
1712 TObjString *str;
1713 while((str=(TObjString*)next())) {
1714 if (str->GetString().Contains(".so"))
1715 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1716 }
1717 if (list) delete list;
1718 }
1719 out << endl;
c57f56b7 1720 out << "// analysis source to be compiled at runtime (if any)" << endl;
1721 if (fAnalysisSource.Length()) {
1722 TObjArray *list = fAnalysisSource.Tokenize(" ");
1723 TIter next(list);
1724 TObjString *str;
1725 while((str=(TObjString*)next())) {
1726 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1727 }
1728 if (list) delete list;
1729 }
1730 out << endl;
1731 out << "// connect to AliEn and make the chain" << endl;
1732 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1733 if (IsUsingTags()) {
1734 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1735 } else {
76535c98 1736 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1737 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1738 } else {
1739 out << " // Check if the macro to create the chain was provided" << endl;
1740 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1741 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1742 out << " return;" << endl;
1743 out << " }" << endl;
1744 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1745 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1746 }
c57f56b7 1747 }
1748 out << "// read the analysis manager from file" << endl;
1749 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1750 out << " if (!file) return;" << endl;
1751 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1752 out << " AliAnalysisManager *mgr = 0;" << endl;
1753 out << " TKey *key;" << endl;
1754 out << " while ((key=(TKey*)nextkey())) {" << endl;
1755 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1756 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1757 out << " };" << endl;
1758 out << " if (!mgr) {" << endl;
1759 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1760 out << " return;" << endl;
1761 out << " }" << endl << endl;
1762 out << " mgr->PrintStatus();" << endl;
1763 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1764 out << " timer.Stop();" << endl;
1765 out << " timer.Print();" << endl;
c57f56b7 1766 out << "}" << endl << endl;
1767 if (IsUsingTags()) {
1768 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1769 out << "{" << endl;
1770 out << "// Create a chain using tags from the xml file." << endl;
1771 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1772 out << " if (!coll) {" << endl;
1773 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1774 out << " return NULL;" << endl;
1775 out << " }" << endl;
1776 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1777 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1778 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1779 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1780 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1781 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1782 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1783 out << " // Check if the cuts configuration file was provided" << endl;
1784 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1785 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1786 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1787 out << " }" << endl;
0df6ccf2 1788 if (fFriendChainName=="") {
1789 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1790 } else {
1791 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1792 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1793 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1794 }
c57f56b7 1795 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1796 out << " chain->ls();" << endl;
1797 out << " return chain;" << endl;
fcc9bb6f 1798 out << "}" << endl << endl;
c57f56b7 1799 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1800 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1801 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1802 msg += " AliLHCTagCuts *lhcCuts,\n";
1803 msg += " AliDetectorTagCuts *detCuts,\n";
1804 msg += " AliEventTagCuts *evCuts)";
1805 Info("WriteAnalysisMacro", msg.Data());
1806 }
0df6ccf2 1807 }
1808 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1809 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1810 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1811 out << "{" << endl;
1812 out << "// Create a chain using url's from xml file" << endl;
1813 out << " TString treename = type;" << endl;
1814 out << " treename.ToLower();" << endl;
1815 out << " treename += \"Tree\";" << endl;
e02fee64 1816 out << " printf(\"***************************************\\n\");" << endl;
1817 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1818 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1819 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1820 out << " if (!coll) {" << endl;
1821 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1822 out << " return NULL;" << endl;
1823 out << " }" << endl;
1824 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1825 if(fFriendChainName!="") {
1826 out << " TChain *chainFriend = new TChain(treename);" << endl;
1827 }
c57f56b7 1828 out << " coll->Reset();" << endl;
0df6ccf2 1829 out << " while (coll->Next()) {" << endl;
1830 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1831 if(fFriendChainName!="") {
1832 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1833 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1834 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1835 out << " chainFriend->Add(fileFriend.Data());" << endl;
1836 }
1837 out << " }" << endl;
c57f56b7 1838 out << " if (!chain->GetNtrees()) {" << endl;
1839 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1840 out << " return NULL;" << endl;
1841 out << " }" << endl;
0df6ccf2 1842 if(fFriendChainName!="") {
1843 out << " chain->AddFriend(chainFriend);" << endl;
1844 }
c57f56b7 1845 out << " return chain;" << endl;
fcc9bb6f 1846 out << "}" << endl << endl;
c57f56b7 1847 }
4e5c5506 1848 if (fPackages) {
fcc9bb6f 1849 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1850 out << "Bool_t SetupPar(const char *package) {" << endl;
1851 out << "// Compile the package and set it up." << endl;
1852 out << " TString pkgdir = package;" << endl;
1853 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1854 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1855 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1856 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1857 out << " // Check for BUILD.sh and execute" << endl;
1858 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1859 out << " printf(\"*******************************\\n\");" << endl;
1860 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1861 out << " printf(\"*******************************\\n\");" << endl;
1862 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1863 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1864 out << " gSystem->ChangeDirectory(cdir);" << endl;
1865 out << " return kFALSE;" << endl;
1866 out << " }" << endl;
1867 out << " } else {" << endl;
fcc9bb6f 1868 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1869 out << " gSystem->ChangeDirectory(cdir);" << endl;
1870 out << " return kFALSE;" << endl;
1871 out << " }" << endl;
1872 out << " // Check for SETUP.C and execute" << endl;
1873 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1874 out << " printf(\"*******************************\\n\");" << endl;
1875 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1876 out << " printf(\"*******************************\\n\");" << endl;
1877 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1878 out << " } else {" << endl;
fcc9bb6f 1879 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1880 out << " gSystem->ChangeDirectory(cdir);" << endl;
1881 out << " return kFALSE;" << endl;
1882 out << " }" << endl;
1883 out << " // Restore original workdir" << endl;
1884 out << " gSystem->ChangeDirectory(cdir);" << endl;
1885 out << " return kTRUE;" << endl;
1886 out << "}" << endl;
1887 }
c57f56b7 1888 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1889 }
1890 Bool_t copy = kTRUE;
1891 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1892 if (copy) {
1893 CdWork();
1894 TString workdir = gGrid->GetHomeDirectory();
1895 workdir += fGridWorkingDir;
1896 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1897 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1898 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1899 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1900 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1901 }
1902 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1903 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1904 }
1905}
1906
1907//______________________________________________________________________________
1908void AliAnalysisAlien::WriteExecutable()
1909{
1910// Generate the alien executable script.
1911 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1912 ofstream out;
1913 out.open(fExecutable.Data(), ios::out);
1914 if (out.bad()) {
5513444a 1915 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1916 return;
1917 }
1918 out << "#!/bin/bash" << endl;
1919 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1920 out << "echo \"=========================================\"" << endl;
1921 out << "echo \"############## PATH : ##############\"" << endl;
1922 out << "echo $PATH" << endl;
1923 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1924 out << "echo $LD_LIBRARY_PATH" << endl;
1925 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1926 out << "echo $ROOTSYS" << endl;
1927 out << "echo \"############## which root : ##############\"" << endl;
1928 out << "which root" << endl;
1929 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1930 out << "echo $ALICE_ROOT" << endl;
1931 out << "echo \"############## which aliroot : ##############\"" << endl;
1932 out << "which aliroot" << endl;
1933 out << "echo \"=========================================\"" << endl << endl;
1934// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1935 out << "root -b -q ";
1936 out << fAnalysisMacro.Data() << endl << endl;
1937 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1938 }
1939 Bool_t copy = kTRUE;
1940 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1941 if (copy) {
1942 CdWork();
1943 TString workdir = gGrid->GetHomeDirectory();
1944 workdir += fGridWorkingDir;
1945 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1946 if (FileExists(executable)) gGrid->Rm(executable);
1947 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1948 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1949 }
1950}
1951
1952//______________________________________________________________________________
5513444a 1953void AliAnalysisAlien::WriteProductionFile(const char *filename) const
1954{
1955// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 1956// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 1957// Next lines: full_path_to_dataset XXX (XXX is a string)
1958// To submit, one has to: submit jdl XXX for all lines
1959 ofstream out;
1960 out.open(filename, ios::out);
1961 if (out.bad()) {
1962 Error("WriteProductionFile", "Bad file name: %s", filename);
1963 return;
1964 }
1965 TString workdir = gGrid->GetHomeDirectory();
1966 workdir += fGridWorkingDir;
f5e8c702 1967 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 1968 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 1969 out << locjdl << " " << njobspermaster << endl;
5513444a 1970 Int_t nmasterjobs = fInputFiles->GetEntries();
1971 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 1972 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 1973 }
1974 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
1975 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
1976}
1977
1978//______________________________________________________________________________
c57f56b7 1979void AliAnalysisAlien::WriteValidationScript()
1980{
1981// Generate the alien validation script.
1982 // Generate the validation script
1983 TObjString *os;
1984 if (!Connect()) {
1985 Error("WriteValidationScript", "Alien connection required");
1986 return;
1987 }
1988 TString out_stream = "";
1989 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1990 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1991 ofstream out;
1992 out.open("validate.sh", ios::out);
1993 out << "#!/bin/bash" << endl;
1994 out << "##################################################" << endl;
1995 out << "validateout=`dirname $0`" << endl;
1996 out << "validatetime=`date`" << endl;
1997 out << "validated=\"0\";" << endl;
1998 out << "error=0" << endl;
1999 out << "if [ -z $validateout ]" << endl;
2000 out << "then" << endl;
2001 out << " validateout=\".\"" << endl;
2002 out << "fi" << endl << endl;
2003 out << "cd $validateout;" << endl;
2004 out << "validateworkdir=`pwd`;" << endl << endl;
2005 out << "echo \"*******************************************************\"" << out_stream << endl;
2006 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2007 out << "" << endl;
2008 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2009 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2010 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2011 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2012 out << "ls -la ./" << out_stream << endl;
2013 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2014 out << "##################################################" << endl;
ebec370a 2015
2016 out << "" << endl;
2017 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2018 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2019 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2020 out << "" << endl;
2021
2022 out << "if [ ! -f stderr ] ; then" << endl;
2023 out << " error=1" << endl;
2024 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2025 out << " echo \"Error = $error\" " << out_stream << endl;
2026 out << "fi" << endl;
2027
2028 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2029 out << " error=1" << endl;
2030 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2031 out << " echo \"$parArch\" " << out_stream << endl;
2032 out << " echo \"Error = $error\" " << out_stream << endl;
2033 out << "fi" << endl;
2034
2035 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2036 out << " error=1" << endl;
2037 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2038 out << " echo \"$segViol\" " << out_stream << endl;
2039 out << " echo \"Error = $error\" " << out_stream << endl;
2040 out << "fi" << endl;
2041
2042 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2043 out << " error=1" << endl;
2044 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2045 out << " echo \"$segFault\" " << out_stream << endl;
2046 out << " echo \"Error = $error\" " << out_stream << endl;
2047 out << "fi" << endl;
2048
2049 // Part dedicated to the specific analyses running into the train
2050
c57f56b7 2051 TObjArray *arr = fOutputFiles.Tokenize(" ");
2052 TIter next1(arr);
2053 TString output_file;
2054 while ((os=(TObjString*)next1())) {
2055 output_file = os->GetString();
2056 Int_t index = output_file.Index("@");
2057 if (index > 0) output_file.Remove(index);
2058 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2059 out << " error=1" << endl;
2060 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2061 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2062 out << "fi" << endl;
2063 }
2064 delete arr;
2065 out << "if [ $error = 0 ] ; then" << endl;
2066 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2067 out << "fi" << endl;
2068
2069 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2070 out << "echo \"*******************************************************\"" << out_stream << endl;
2071 out << "cd -" << endl;
2072 out << "exit $error" << endl;
2073 }
2074 Bool_t copy = kTRUE;
2075 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2076 if (copy) {
2077 CdWork();
2078 TString workdir = gGrid->GetHomeDirectory();
2079 workdir += fGridWorkingDir;
2080 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
2081 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
2082 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
2083 }
2084}