]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
A new function returning the unconstrained primary vertex (A. Dainese)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
c57f56b7 56 fRunNumbers(),
57 fExecutable(),
58 fArguments(),
59 fAnalysisMacro(),
60 fAnalysisSource(),
61 fAdditionalLibs(),
62 fSplitMode(),
63 fAPIVersion(),
64 fROOTVersion(),
65 fAliROOTVersion(),
66 fUser(),
67 fGridWorkingDir(),
68 fGridDataDir(),
69 fDataPattern(),
70 fGridOutputDir(),
71 fOutputArchive(),
72 fOutputFiles(),
73 fInputFormat(),
e7c71df0 74 fDatasetName(),
c57f56b7 75 fJDLName(),
bb885a9e 76 fMergeExcludes(),
f965131e 77 fIncludePath(),
bb885a9e 78 fCloseSE(),
0df6ccf2 79 fFriendChainName(),
4e5c5506 80 fInputFiles(0),
81 fPackages(0)
c57f56b7 82{
83// Dummy ctor.
84 SetDefaults();
85}
86
87//______________________________________________________________________________
88AliAnalysisAlien::AliAnalysisAlien(const char *name)
89 :AliAnalysisGrid(name),
90 fGridJDL(NULL),
91 fPrice(0),
92 fTTL(0),
93 fSplitMaxInputFileNumber(0),
94 fMaxInitFailed(0),
95 fMasterResubmitThreshold(0),
bb885a9e 96 fNtestFiles(0),
319593fb 97 fNrunsPerMaster(0),
16a4353c 98 fMaxMergeFiles(0),
d2a409b2 99 fNsubmitted(0),
c57f56b7 100 fRunNumbers(),
101 fExecutable(),
102 fArguments(),
103 fAnalysisMacro(),
104 fAnalysisSource(),
105 fAdditionalLibs(),
106 fSplitMode(),
107 fAPIVersion(),
108 fROOTVersion(),
109 fAliROOTVersion(),
110 fUser(),
111 fGridWorkingDir(),
112 fGridDataDir(),
113 fDataPattern(),
114 fGridOutputDir(),
115 fOutputArchive(),
116 fOutputFiles(),
117 fInputFormat(),
e7c71df0 118 fDatasetName(),
c57f56b7 119 fJDLName(),
bb885a9e 120 fMergeExcludes(),
f965131e 121 fIncludePath(),
bb885a9e 122 fCloseSE(),
0df6ccf2 123 fFriendChainName(),
4e5c5506 124 fInputFiles(0),
125 fPackages(0)
c57f56b7 126{
127// Default ctor.
128 SetDefaults();
129}
130
131//______________________________________________________________________________
132AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
133 :AliAnalysisGrid(other),
134 fGridJDL(NULL),
135 fPrice(other.fPrice),
136 fTTL(other.fTTL),
137 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
138 fMaxInitFailed(other.fMaxInitFailed),
139 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 140 fNtestFiles(other.fNtestFiles),
319593fb 141 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 142 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 143 fNsubmitted(other.fNsubmitted),
c57f56b7 144 fRunNumbers(other.fRunNumbers),
145 fExecutable(other.fExecutable),
146 fArguments(other.fArguments),
147 fAnalysisMacro(other.fAnalysisMacro),
148 fAnalysisSource(other.fAnalysisSource),
149 fAdditionalLibs(other.fAdditionalLibs),
150 fSplitMode(other.fSplitMode),
151 fAPIVersion(other.fAPIVersion),
152 fROOTVersion(other.fROOTVersion),
153 fAliROOTVersion(other.fAliROOTVersion),
154 fUser(other.fUser),
155 fGridWorkingDir(other.fGridWorkingDir),
156 fGridDataDir(other.fGridDataDir),
157 fDataPattern(other.fDataPattern),
158 fGridOutputDir(other.fGridOutputDir),
159 fOutputArchive(other.fOutputArchive),
160 fOutputFiles(other.fOutputFiles),
161 fInputFormat(other.fInputFormat),
e7c71df0 162 fDatasetName(other.fDatasetName),
c57f56b7 163 fJDLName(other.fJDLName),
bb885a9e 164 fMergeExcludes(other.fMergeExcludes),
f965131e 165 fIncludePath(other.fIncludePath),
bb885a9e 166 fCloseSE(other.fCloseSE),
0df6ccf2 167 fFriendChainName(other.fFriendChainName),
4e5c5506 168 fInputFiles(0),
169 fPackages(0)
c57f56b7 170{
171// Copy ctor.
172 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 173 fRunRange[0] = other.fRunRange[0];
174 fRunRange[1] = other.fRunRange[1];
c57f56b7 175 if (other.fInputFiles) {
176 fInputFiles = new TObjArray();
177 TIter next(other.fInputFiles);
178 TObject *obj;
179 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
180 fInputFiles->SetOwner();
181 }
4e5c5506 182 if (other.fPackages) {
183 fPackages = new TObjArray();
184 TIter next(other.fPackages);
185 TObject *obj;
186 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
187 fPackages->SetOwner();
188 }
c57f56b7 189}
190
191//______________________________________________________________________________
192AliAnalysisAlien::~AliAnalysisAlien()
193{
194// Destructor.
195 if (fGridJDL) delete fGridJDL;
196 if (fInputFiles) delete fInputFiles;
4e5c5506 197 if (fPackages) delete fPackages;
c57f56b7 198}
199
200//______________________________________________________________________________
201AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
202{
203// Assignment.
204 if (this != &other) {
205 AliAnalysisGrid::operator=(other);
206 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
207 fPrice = other.fPrice;
208 fTTL = other.fTTL;
209 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
210 fMaxInitFailed = other.fMaxInitFailed;
211 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 212 fNtestFiles = other.fNtestFiles;
c57f56b7 213 fRunNumbers = other.fRunNumbers;
214 fExecutable = other.fExecutable;
215 fArguments = other.fArguments;
216 fAnalysisMacro = other.fAnalysisMacro;
217 fAnalysisSource = other.fAnalysisSource;
218 fAdditionalLibs = other.fAdditionalLibs;
219 fSplitMode = other.fSplitMode;
220 fAPIVersion = other.fAPIVersion;
221 fROOTVersion = other.fROOTVersion;
222 fAliROOTVersion = other.fAliROOTVersion;
223 fUser = other.fUser;
224 fGridWorkingDir = other.fGridWorkingDir;
225 fGridDataDir = other.fGridDataDir;
226 fDataPattern = other.fDataPattern;
227 fGridOutputDir = other.fGridOutputDir;
228 fOutputArchive = other.fOutputArchive;
229 fOutputFiles = other.fOutputFiles;
230 fInputFormat = other.fInputFormat;
e7c71df0 231 fDatasetName = other.fDatasetName;
c57f56b7 232 fJDLName = other.fJDLName;
bb885a9e 233 fMergeExcludes = other.fMergeExcludes;
f965131e 234 fIncludePath = other.fIncludePath;
bb885a9e 235 fCloseSE = other.fCloseSE;
0df6ccf2 236 fFriendChainName = other.fFriendChainName;
c57f56b7 237 if (other.fInputFiles) {
238 fInputFiles = new TObjArray();
239 TIter next(other.fInputFiles);
240 TObject *obj;
241 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
242 fInputFiles->SetOwner();
243 }
4e5c5506 244 if (other.fPackages) {
245 fPackages = new TObjArray();
246 TIter next(other.fPackages);
247 TObject *obj;
248 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
249 fPackages->SetOwner();
250 }
c57f56b7 251 }
252 return *this;
253}
254
f965131e 255//______________________________________________________________________________
256void AliAnalysisAlien::AddIncludePath(const char *path)
257{
258// Add include path in the remote analysis macro.
259 TString p(path);
260 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
261 else fIncludePath += Form("-I%s ", path);
262}
263
c57f56b7 264//______________________________________________________________________________
265void AliAnalysisAlien::AddRunNumber(Int_t run)
266{
267// Add a run number to the list of runs to be processed.
268 if (fRunNumbers.Length()) fRunNumbers += " ";
269 fRunNumbers += Form("%d", run);
270}
271
272//______________________________________________________________________________
273void AliAnalysisAlien::AddDataFile(const char *lfn)
274{
275// Adds a data file to the input to be analysed. The file should be a valid LFN
276// or point to an existing file in the alien workdir.
277 if (!fInputFiles) fInputFiles = new TObjArray();
278 fInputFiles->Add(new TObjString(lfn));
279}
280
281//______________________________________________________________________________
282Bool_t AliAnalysisAlien::Connect()
283{
284// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
285 if (gGrid && gGrid->IsConnected()) return kTRUE;
286 if (!gSystem->Getenv("alien_API_USER")) {
287 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
288 gSystem->Getenv("UID"));
289 return kFALSE;
290 }
291 if (!gGrid) {
292 Info("Connect", "Trying to connect to AliEn ...");
293 TGrid::Connect("alien://");
294 }
295 if (!gGrid || !gGrid->IsConnected()) {
296 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
297 return kFALSE;
298 }
299 fUser = gGrid->GetUser();
300 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
301 return kTRUE;
302}
303
304//______________________________________________________________________________
305void AliAnalysisAlien::CdWork()
306{
307// Check validity of alien workspace. Create directory if possible.
308 if (!Connect()) {
309 Error("CdWork", "Alien connection required");
310 return;
311 }
312 TString homedir = gGrid->GetHomeDirectory();
313 TString workdir = homedir + fGridWorkingDir;
314 if (!gGrid->Cd(workdir)) {
315 gGrid->Cd(homedir);
316 if (gGrid->Mkdir(workdir)) {
317 gGrid->Cd(fGridWorkingDir);
318 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
319 } else {
320 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
321 workdir.Data(), homedir.Data());
322 fGridWorkingDir = "";
323 }
324 }
325}
326
327//______________________________________________________________________________
328Bool_t AliAnalysisAlien::CheckInputData()
329{
330// Check validity of input data. If necessary, create xml files.
a8739e8a 331 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 332 if (!fGridDataDir.Length()) {
333 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
334 return kFALSE;
335 }
336 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
337 return kTRUE;
c57f56b7 338 }
339 // Process declared files
340 Bool_t is_collection = kFALSE;
341 Bool_t is_xml = kFALSE;
342 Bool_t use_tags = kFALSE;
343 Bool_t checked = kFALSE;
344 CdWork();
345 TString file;
346 TString workdir = gGrid->GetHomeDirectory();
347 workdir += fGridWorkingDir;
348 if (fInputFiles) {
349 TObjString *objstr;
350 TIter next(fInputFiles);
351 while ((objstr=(TObjString*)next())) {
352 file = workdir;
353 file += "/";
354 file += objstr->GetString();
355 // Store full lfn path
356 if (FileExists(file)) objstr->SetString(file);
357 else {
358 file = objstr->GetName();
359 if (!FileExists(objstr->GetName())) {
360 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
361 objstr->GetName(), workdir.Data());
362 return kFALSE;
363 }
364 }
365 Bool_t iscoll, isxml, usetags;
366 CheckDataType(file, iscoll, isxml, usetags);
367 if (!checked) {
368 checked = kTRUE;
369 is_collection = iscoll;
370 is_xml = isxml;
371 use_tags = usetags;
372 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
373 } else {
374 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
375 Error("CheckInputData", "Some conflict was found in the types of inputs");
376 return kFALSE;
377 }
378 }
379 }
380 }
381 // Process requested run numbers
a8739e8a 382 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 383 // Check validity of alien data directory
384 if (!fGridDataDir.Length()) {
385 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
386 return kFALSE;
387 }
388 if (!gGrid->Cd(fGridDataDir)) {
389 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
390 return kFALSE;
391 }
392 if (is_collection) {
393 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
394 return kFALSE;
395 }
396
397 if (checked && !is_xml) {
398 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
399 return kFALSE;
400 }
401 // Check validity of run number(s)
402 TObjArray *arr;
403 TObjString *os;
319593fb 404 Int_t nruns = 0;
405 TString schunk;
c57f56b7 406 TString path;
407 if (!checked) {
408 checked = kTRUE;
409 use_tags = fDataPattern.Contains("tag");
410 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
411 }
412 if (use_tags != fDataPattern.Contains("tag")) {
413 Error("CheckInputData", "Cannot mix input files using/not using tags");
414 return kFALSE;
415 }
416 if (fRunNumbers.Length()) {
a8739e8a 417 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 418 arr = fRunNumbers.Tokenize(" ");
419 TIter next(arr);
420 while ((os=(TObjString*)next())) {
421 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
422 if (!gGrid->Cd(path)) {
a8739e8a 423 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
424 continue;
c57f56b7 425 }
426 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
427 TString msg = "\n##### file: ";
428 msg += path;
429 msg += " type: xml_collection;";
430 if (use_tags) msg += " using_tags: Yes";
431 else msg += " using_tags: No";
432 Info("CheckDataType", msg.Data());
319593fb 433 if (fNrunsPerMaster<2) {
d2a409b2 434 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 435 } else {
436 nruns++;
437 if (((nruns-1)%fNrunsPerMaster) == 0) {
438 schunk = os->GetString();
439 }
440 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
441 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 442 AddDataFile(schunk);
319593fb 443 }
c57f56b7 444 }
445 delete arr;
a8739e8a 446 } else {
447 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
448 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
449 path = Form("%s/%d ", fGridDataDir.Data(), irun);
450 if (!gGrid->Cd(path)) {
451 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
452 continue;
453 }
454 path = Form("%s/%d.xml", workdir.Data(),irun);
455 TString msg = "\n##### file: ";
456 msg += path;
457 msg += " type: xml_collection;";
458 if (use_tags) msg += " using_tags: Yes";
459 else msg += " using_tags: No";
460 Info("CheckDataType", msg.Data());
319593fb 461 if (fNrunsPerMaster<2) {
d2a409b2 462 AddDataFile(Form("%d.xml",irun));
319593fb 463 } else {
464 nruns++;
465 if (((nruns-1)%fNrunsPerMaster) == 0) {
466 schunk = Form("%d", irun);
467 }
468 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
469 schunk += Form("_%d.xml", irun);
d2a409b2 470 AddDataFile(schunk);
319593fb 471 }
a8739e8a 472 }
c57f56b7 473 }
474 return kTRUE;
475}
476
477//______________________________________________________________________________
478Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
479{
480// Create dataset for the grid data directory + run number.
481 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
482 if (!Connect()) {
483 Error("CreateDataset", "Cannot create dataset with no grid connection");
484 return kFALSE;
485 }
486
487 // Cd workspace
488 CdWork();
489 TString workdir = gGrid->GetHomeDirectory();
490 workdir += fGridWorkingDir;
491
492 // Compose the 'find' command arguments
493 TString command;
494 TString options = "-x collection ";
bb885a9e 495 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 496 TString conditions = "";
497
498 TString file;
499 TString path;
319593fb 500 Int_t nruns = 0;
501 TString schunk;
ab254fd1 502 TGridCollection *cbase=0, *cadd=0;
d2a409b2 503 if (!fRunNumbers.Length() && !fRunRange[0]) {
504 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
505 // Make a single data collection from data directory.
506 path = fGridDataDir;
507 if (!gGrid->Cd(path)) {
508 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
509 return kFALSE;
510 }
511 CdWork();
512 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
513 else file = Form("%s.xml", gSystem->BaseName(path));
514 if (gSystem->AccessPathName(file)) {
515 command = "find ";
516 command += options;
517 command += path;
518 command += " ";
519 command += pattern;
520 command += conditions;
521 printf("command: %s\n", command.Data());
522 TGridResult *res = gGrid->Command(command);
523 if (res) delete res;
524 // Write standard output to file
525 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
526 }
527 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
528 // Copy xml file to alien space
529 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
530 if (!FileExists(file)) {
531 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
532 return kFALSE;
533 }
534 // Update list of files to be processed.
535 }
536 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
537 return kTRUE;
538 }
c57f56b7 539 // Several runs
a8739e8a 540 if (fRunNumbers.Length()) {
541 TObjArray *arr = fRunNumbers.Tokenize(" ");
542 TObjString *os;
543 TIter next(arr);
544 while ((os=(TObjString*)next())) {
545 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
673d4fb3 546 if (!gGrid->Cd(path)) continue;
95e5b448 547 CdWork();
a8739e8a 548 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
549 else file = Form("%s.xml", os->GetString().Data());
319593fb 550 // If local collection file does not exist, create it via 'find' command.
551 if (gSystem->AccessPathName(file)) {
552 command = "find ";
553 command += options;
554 command += path;
555 command += pattern;
556 command += conditions;
557 TGridResult *res = gGrid->Command(command);
558 if (res) delete res;
559 // Write standard output to file
560 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
561 }
a8739e8a 562 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 563 // Check if there is one run per master job.
564 if (fNrunsPerMaster<2) {
565 if (FileExists(file)) {
566 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
567 continue;
568 }
569 // Copy xml file to alien space
570 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
571 if (!FileExists(file)) {
572 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
573 delete arr;
574 return kFALSE;
575 }
576 } else {
577 nruns++;
578 if (((nruns-1)%fNrunsPerMaster) == 0) {
579 schunk = os->GetString();
580 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
581 } else {
582 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
583 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
584 cbase->Add(cadd);
585 delete cadd;
586 }
587 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
588 continue;
589 }
590 schunk += Form("_%s.xml", os->GetString().Data());
591 if (FileExists(schunk)) {
592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
593 continue;
594 }
595 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
596 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
597 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
598 if (!FileExists(schunk)) {
599 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
600 delete arr;
601 return kFALSE;
602 }
603 }
a8739e8a 604 }
605 delete arr;
606 } else {
607 // Process a full run range.
608 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
609 path = Form("%s/%d ", fGridDataDir.Data(), irun);
673d4fb3 610 if (!gGrid->Cd(path)) continue;
95e5b448 611 CdWork();
a8739e8a 612 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
613 else file = Form("%d.xml", irun);
319593fb 614 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 615 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
616// gGrid->Rm(file);
617 continue;
618 }
319593fb 619 // If local collection file does not exist, create it via 'find' command.
620 if (gSystem->AccessPathName(file)) {
621 command = "find ";
622 command += options;
623 command += path;
624 command += pattern;
625 command += conditions;
626 TGridResult *res = gGrid->Command(command);
627 if (res) delete res;
628 // Write standard output to file
629 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
630 }
a8739e8a 631 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 632 // Check if there is one run per master job.
633 if (fNrunsPerMaster<2) {
634 if (FileExists(file)) {
635 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
636 continue;
637 }
638 // Copy xml file to alien space
639 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
640 if (!FileExists(file)) {
641 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
642 return kFALSE;
643 }
644 } else {
645 nruns++;
95e5b448 646 // Check if the collection for the chunk exist locally.
647 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
648 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
319593fb 649 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
650 if (((nruns-1)%fNrunsPerMaster) == 0) {
651 schunk = Form("%d", irun);
652 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
653 } else {
654 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
655 cbase->Add(cadd);
656 delete cadd;
657 }
658 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
659 continue;
660 }
661 schunk += Form("_%d.xml", irun);
662 if (FileExists(schunk)) {
663 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
664 continue;
665 }
666 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
667 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 668 if (FileExists(schunk)) {
669 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
670 continue;
671 }
319593fb 672 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
673 if (!FileExists(schunk)) {
674 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
675 return kFALSE;
676 }
677 }
c57f56b7 678 }
a8739e8a 679 }
c57f56b7 680 return kTRUE;
681}
682
683//______________________________________________________________________________
684Bool_t AliAnalysisAlien::CreateJDL()
685{
686// Generate a JDL file according to current settings. The name of the file is
687// specified by fJDLName.
688 Bool_t error = kFALSE;
689 TObjArray *arr = 0;
690 Bool_t copy = kTRUE;
691 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
692 Bool_t generate = kTRUE;
693 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
694 if (!Connect()) {
695 Error("CreateJDL", "Alien connection required");
696 return kFALSE;
697 }
698 // Check validity of alien workspace
699 CdWork();
700 TString workdir = gGrid->GetHomeDirectory();
701 workdir += fGridWorkingDir;
702 if (generate) {
703 TObjString *os;
704 if (!fInputFiles) {
705 Error("CreateJDL()", "Define some input files for your analysis.");
706 error = kTRUE;
707 }
708 // Compose list of input files
709 // Check if output files were defined
710 if (!fOutputFiles.Length()) {
711 Error("CreateJDL", "You must define at least one output file");
712 error = kTRUE;
713 }
714 // Check if an output directory was defined and valid
715 if (!fGridOutputDir.Length()) {
716 Error("CreateJDL", "You must define AliEn output directory");
717 error = kTRUE;
718 } else {
d2a409b2 719 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
c57f56b7 720 if (!gGrid->Cd(fGridOutputDir)) {
721 if (gGrid->Mkdir(fGridOutputDir)) {
722 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
723 } else {
724 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 725 // error = kTRUE;
c57f56b7 726 }
727 }
728 gGrid->Cd(workdir);
729 }
730 // Exit if any error up to now
731 if (error) return kFALSE;
732 // Set JDL fields
733 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
734 fGridJDL->SetExecutable(fExecutable);
735// fGridJDL->SetTTL((UInt_t)fTTL);
736 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
737 if (fMaxInitFailed > 0)
738 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
739 if (fSplitMaxInputFileNumber > 0)
740 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
741 if (fSplitMode.Length())
742 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
743// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
744 if (fAliROOTVersion.Length())
745 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
746 if (fROOTVersion.Length())
747 fGridJDL->AddToPackages("ROOT", fROOTVersion);
748 if (fAPIVersion.Length())
749 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
750 fGridJDL->SetInputDataListFormat(fInputFormat);
751 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 752 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
753 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
754 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
755 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
756 if (fAdditionalLibs.Length()) {
757 arr = fAdditionalLibs.Tokenize(" ");
758 TIter next(arr);
759 while ((os=(TObjString*)next())) {
760 if (os->GetString().Contains(".so")) continue;
761 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
762 }
763 delete arr;
764 }
4e5c5506 765 if (fPackages) {
766 TIter next(fPackages);
767 TObject *obj;
768 while ((obj=next()))
769 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
770 }
c57f56b7 771 if (fOutputArchive.Length()) {
772 arr = fOutputArchive.Tokenize(" ");
773 TIter next(arr);
774 while ((os=(TObjString*)next()))
43da816a 775 if (!os->GetString().Contains("@") && fCloseSE.Length())
776 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
777 else
778 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 779 delete arr;
780 }
c57f56b7 781 arr = fOutputFiles.Tokenize(" ");
782 TIter next(arr);
43da816a 783 while ((os=(TObjString*)next())) {
784 if (!os->GetString().Contains("@") && fCloseSE.Length())
785 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
786 else
787 fGridJDL->AddToOutputSandbox(os->GetString());
788 }
c57f56b7 789 delete arr;
790// fGridJDL->SetPrice((UInt_t)fPrice);
791 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
792 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
793 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 794 // Write a jdl with 2 input parameters: collection name and output dir name.
795 WriteJDL(copy);
c57f56b7 796 }
797 // Copy jdl to grid workspace
a8739e8a 798 if (copy) {
c57f56b7 799 if (fAdditionalLibs.Length()) {
e7c71df0 800 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 801 TObjString *os;
802 TIter next(arr);
803 while ((os=(TObjString*)next())) {
c57f56b7 804 if (os->GetString().Contains(".so")) continue;
4e5c5506 805 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 806 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
807 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
808 }
809 delete arr;
810 }
4e5c5506 811 if (fPackages) {
812 TIter next(fPackages);
813 TObject *obj;
814 while ((obj=next())) {
815 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
816 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
817 }
818 }
c57f56b7 819 }
820 return kTRUE;
821}
822
a8739e8a 823//______________________________________________________________________________
d2a409b2 824Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 825{
826// Writes one or more JDL's corresponding to findex. If findex is negative,
827// all run numbers are considered in one go (jdl). For non-negative indices
828// they correspond to the indices in the array fInputFiles.
829 if (!fInputFiles) return kFALSE;
830 TObjString *os;
a8739e8a 831 TString workdir = gGrid->GetHomeDirectory();
832 workdir += fGridWorkingDir;
d2a409b2 833
834 if (!fRunNumbers.Length() && !fRunRange[0]) {
835 // One jdl with no parameters in case input data is specified by name.
a8739e8a 836 TIter next(fInputFiles);
837 while ((os=(TObjString*)next()))
838 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 839 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 840 } else {
d2a409b2 841 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
842 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
843 fGridJDL->SetOutputDirectory(Form("%s/$2#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 844 }
845
846
847 // Generate the JDL as a string
848 TString sjdl = fGridJDL->Generate();
849 Int_t index;
850 index = sjdl.Index("Executable");
851 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
852 index = sjdl.Index("Split ");
853 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
854 index = sjdl.Index("SplitMaxInputFileNumber");
855 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
856 index = sjdl.Index("InputDataCollection");
857 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 858 index = sjdl.Index("InputFile");
859 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
860 index = sjdl.Index("InputDataList ");
861 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
862 index = sjdl.Index("InputDataListFormat");
863 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
864 index = sjdl.Index("Price");
865 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
866 index = sjdl.Index("Requirements");
867 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
868 index = sjdl.Index("Packages");
869 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
870 index = sjdl.Index("User =");
871 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
872 index = sjdl.Index("TTL");
873 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
874 index = sjdl.Index("OutputFile");
875 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
876 index = sjdl.Index("OutputDir");
877 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
878 index = sjdl.Index("OutputArchive");
879 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
880 index = sjdl.Index("MaxInitFailed");
881 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
882 index = sjdl.Index("MasterResubmitThreshold");
883 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
884 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
885 index = sjdl.Index("Validationcommand");
886 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
887 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
888 sjdl.ReplaceAll("(member", "\n (member");
889 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
890 sjdl.ReplaceAll("{", "{\n ");
891 sjdl.ReplaceAll("};", "\n};");
892 sjdl.ReplaceAll("{\n \n", "{\n");
893 sjdl.ReplaceAll("\n\n", "\n");
894 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
895 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
896 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
897 index = sjdl.Index("JDLVariables");
898 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
899 // Write jdl to file
a8739e8a 900 ofstream out;
d2a409b2 901 out.open(fJDLName.Data(), ios::out);
a8739e8a 902 if (out.bad()) {
d2a409b2 903 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 904 return kFALSE;
905 }
906 out << sjdl << endl;
907
908 // Copy jdl to grid workspace
909 if (!copy) {
910 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
911 } else {
d2a409b2 912 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
913 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
5513444a 914 if (TObject::TestBit(AliAnalysisGrid::kProductionMode))
b5fe9cba 915 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 916 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 917 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 918 }
919 return kTRUE;
920}
921
c57f56b7 922//______________________________________________________________________________
5513444a 923Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 924{
925// Returns true if file exists.
5513444a 926 if (!gGrid) return kFALSE;
c57f56b7 927 TGridResult *res = gGrid->Ls(lfn);
928 if (!res) return kFALSE;
929 TMap *map = dynamic_cast<TMap*>(res->At(0));
930 if (!map) {
931 delete res;
932 return kFALSE;
933 }
934 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
935 if (!objs || !objs->GetString().Length()) {
936 delete res;
937 return kFALSE;
938 }
939 delete res;
940 return kTRUE;
941}
942
943//______________________________________________________________________________
944void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
945{
946// Check input data type.
947 is_collection = kFALSE;
948 is_xml = kFALSE;
949 use_tags = kFALSE;
950 if (!gGrid) {
951 Error("CheckDataType", "No connection to grid");
952 return;
953 }
954 is_collection = IsCollection(lfn);
955 TString msg = "\n##### file: ";
956 msg += lfn;
957 if (is_collection) {
958 msg += " type: raw_collection;";
959 // special treatment for collections
960 is_xml = kFALSE;
961 // check for tag files in the collection
962 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
963 if (!res) {
964 msg += " using_tags: No (unknown)";
965 Info("CheckDataType", msg.Data());
966 return;
967 }
968 const char* typeStr = res->GetKey(0, "origLFN");
969 if (!typeStr || !strlen(typeStr)) {
970 msg += " using_tags: No (unknown)";
971 Info("CheckDataType", msg.Data());
972 return;
973 }
974 TString file = typeStr;
975 use_tags = file.Contains(".tag");
976 if (use_tags) msg += " using_tags: Yes";
977 else msg += " using_tags: No";
978 Info("CheckDataType", msg.Data());
979 return;
980 }
981 TString slfn(lfn);
982 slfn.ToLower();
983 is_xml = slfn.Contains(".xml");
984 if (is_xml) {
985 // Open xml collection and check if there are tag files inside
986 msg += " type: xml_collection;";
987 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
988 if (!coll) {
989 msg += " using_tags: No (unknown)";
990 Info("CheckDataType", msg.Data());
991 return;
992 }
993 TMap *map = coll->Next();
994 if (!map) {
995 msg += " using_tags: No (unknown)";
996 Info("CheckDataType", msg.Data());
997 return;
998 }
999 map = (TMap*)map->GetValue("");
1000 TString file;
1001 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1002 use_tags = file.Contains(".tag");
1003 delete coll;
1004 if (use_tags) msg += " using_tags: Yes";
1005 else msg += " using_tags: No";
1006 Info("CheckDataType", msg.Data());
1007 return;
1008 }
1009 use_tags = slfn.Contains(".tag");
1010 if (slfn.Contains(".root")) msg += " type: root file;";
1011 else msg += " type: unhnown file;";
1012 if (use_tags) msg += " using_tags: Yes";
1013 else msg += " using_tags: No";
1014 Info("CheckDataType", msg.Data());
1015}
1016
4e5c5506 1017//______________________________________________________________________________
1018void AliAnalysisAlien::EnablePackage(const char *package)
1019{
1020// Enables a par file supposed to exist in the current directory.
1021 TString pkg(package);
1022 pkg.ReplaceAll(".par", "");
1023 pkg += ".par";
1024 if (gSystem->AccessPathName(pkg)) {
1025 Error("EnablePackage", "Package %s not found", pkg.Data());
1026 return;
1027 }
1028 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1029 Info("EnablePackage", "AliEn plugin will use .par packages");
1030 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1031 if (!fPackages) {
1032 fPackages = new TObjArray();
1033 fPackages->SetOwner();
1034 }
1035 fPackages->Add(new TObjString(pkg));
1036}
1037
d2a409b2 1038//______________________________________________________________________________
1039const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1040{
1041// Get job status for all jobs with jobid>jobidstart.
1042 static char mstatus[20];
1043 mstatus[0] = '\0';
1044 nrunning = 0;
1045 nwaiting = 0;
1046 nerror = 0;
1047 ndone = 0;
1048 TGridJobStatusList *list = gGrid->Ps("");
1049 if (!list) return mstatus;
1050 Int_t nentries = list->GetSize();
1051 TGridJobStatus *status;
1052 Int_t pid;
1053 for (Int_t ijob=0; ijob<nentries; ijob++) {
1054 status = (TGridJobStatus *)list->At(ijob);
1055 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1056 if (pid<jobidstart) continue;
1057 if (pid == lastid) {
1058 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1059 }
1060 switch (status->GetStatus()) {
1061 case TGridJobStatus::kWAITING:
1062 nwaiting++; break;
1063 case TGridJobStatus::kRUNNING:
1064 nrunning++; break;
1065 case TGridJobStatus::kABORTED:
1066 case TGridJobStatus::kFAIL:
1067 case TGridJobStatus::kUNKNOWN:
1068 nerror++; break;
1069 case TGridJobStatus::kDONE:
1070 ndone++;
1071 }
1072 }
1073 list->Delete();
1074 delete list;
1075 return mstatus;
1076}
1077
c57f56b7 1078//______________________________________________________________________________
1079Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1080{
1081// Returns true if file is a collection. Functionality duplicated from
1082// TAlien::Type() because we don't want to directly depend on TAlien.
1083 if (!gGrid) {
1084 Error("IsCollection", "No connection to grid");
1085 return kFALSE;
1086 }
1087 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1088 if (!res) return kFALSE;
1089 const char* typeStr = res->GetKey(0, "type");
1090 if (!typeStr || !strlen(typeStr)) return kFALSE;
1091 if (!strcmp(typeStr, "collection")) return kTRUE;
1092 delete res;
1093 return kFALSE;
1094}
1095
16a4353c 1096//______________________________________________________________________________
1097void AliAnalysisAlien::Print(Option_t *) const
1098{
1099// Print current plugin settings.
1100 printf("### AliEn analysis plugin current settings ###\n");
1101 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1102 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1103 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1104 if (fUser.Length())
1105 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1106 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1107 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1108 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1109 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1110 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1111 if (fRunNumbers.Length())
1112 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1113 if (fRunRange[0])
1114 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1115 if (!fRunRange[0] && !fRunNumbers.Length()) {
1116 TIter next(fInputFiles);
1117 TObject *obj;
1118 TString list;
1119 while ((obj=next())) list += obj->GetName();
1120 printf("= Input files to be processed: _________________ %s\n", list.Data());
1121 }
1122 if (TestBit(AliAnalysisGrid::kTest))
1123 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1124 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1125 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1126 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1127 printf("=====================================================================\n");
1128 printf("= Job price: ___________________________________ %d\n", fPrice);
1129 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1130 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1131 if (fMaxInitFailed>0)
1132 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1133 if (fMasterResubmitThreshold>0)
1134 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1135 if (fNrunsPerMaster>0)
1136 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
16a4353c 1137 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1138 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
1139 if (fArguments.Length())
1140 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1141 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1142 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1143 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1144 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1145 if (fDatasetName)
1146 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1147 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1148 if (fIncludePath.Data())
1149 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1150 if (fCloseSE.Length())
1151 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1152 if (fFriendChainName.Length())
1153 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1154 if (fPackages) {
1155 TIter next(fPackages);
1156 TObject *obj;
1157 TString list;
1158 while ((obj=next())) list += obj->GetName();
1159 printf("= Par files to be used: ________________________ %s\n", list.Data());
1160 }
1161}
1162
c57f56b7 1163//______________________________________________________________________________
1164void AliAnalysisAlien::SetDefaults()
1165{
1166// Set default values for everything. What cannot be filled will be left empty.
1167 if (fGridJDL) delete fGridJDL;
1168 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1169 fPrice = 1;
1170 fTTL = 30000;
1171 fSplitMaxInputFileNumber = 100;
1172 fMaxInitFailed = 0;
1173 fMasterResubmitThreshold = 0;
bb885a9e 1174 fNtestFiles = 10;
a8739e8a 1175 fRunRange[0] = 0;
1176 fRunRange[1] = 0;
319593fb 1177 fNrunsPerMaster = 1;
16a4353c 1178 fMaxMergeFiles = 100;
c57f56b7 1179 fRunNumbers = "";
1180 fExecutable = "analysis.sh";
1181 fArguments = "";
1182 fAnalysisMacro = "myAnalysis.C";
1183 fAnalysisSource = "";
1184 fAdditionalLibs = "";
1185 fSplitMode = "se";
1186 fAPIVersion = "";
1187 fROOTVersion = "";
1188 fAliROOTVersion = "";
1189 fUser = ""; // Your alien user name
1190 fGridWorkingDir = "";
1191 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1192 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1193 fFriendChainName = "";
c57f56b7 1194 fGridOutputDir = "output";
1195 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1196 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1197 fInputFormat = "xml-single";
1198 fJDLName = "analysis.jdl";
bb885a9e 1199 fMergeExcludes = "";
c57f56b7 1200}
1201
1202//______________________________________________________________________________
1203Bool_t AliAnalysisAlien::MergeOutputs()
1204{
1205// Merge analysis outputs existing in the AliEn space.
1206 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1207 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1208 if (!Connect()) {
1209 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1210 return kFALSE;
1211 }
1212 // Get the output path
d2a409b2 1213 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1214 if (!gGrid->Cd(fGridOutputDir)) {
c57f56b7 1215 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1216 return kFALSE;
1217 }
1218 if (!fOutputFiles.Length()) {
1219 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1220 return kFALSE;
1221 }
1222 TObjArray *list = fOutputFiles.Tokenize(" ");
1223 TIter next(list);
1224 TObjString *str;
1225 TString command;
1226 TString output_file;
16a4353c 1227 TString output_chunk;
1228 TString previous_chunk;
1229 Int_t count_chunk = 0;
1230 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1231 Bool_t merged = kTRUE;
1232 while((str=(TObjString*)next())) {
1233 output_file = str->GetString();
1234 Int_t index = output_file.Index("@");
1235 if (index > 0) output_file.Remove(index);
319593fb 1236 // Skip already merged outputs
1237 if (!gSystem->AccessPathName(output_file)) {
1238 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1239 continue;
1240 }
bb885a9e 1241 if (fMergeExcludes.Length() &&
1242 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1243 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1244 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
c57f56b7 1245 printf("command: %s\n", command.Data());
1246 TGridResult *res = gGrid->Command(command);
1247 if (!res) continue;
1248 TFileMerger *fm = 0;
1249 TIter nextmap(res);
ab254fd1 1250 TMap *map = 0;
16a4353c 1251 previous_chunk = "";
1252 count_chunk = 0;
319593fb 1253 // Check if there is a merge operation to resume
1254 output_chunk = output_file;
1255 output_chunk.ReplaceAll(".root", "_*.root");
1256 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1257 while (1) {
1258 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1259 if (!map) {
1260 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1261 delete res;
1262 return kFALSE;
1263 }
1264 output_chunk = output_file;
1265 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1266 printf("%s\n", output_chunk.Data());
1267 count_chunk++;
1268 if (gSystem->AccessPathName(output_chunk)) continue;
1269 // Merged file with chunks up to <count_chunk> found
1270 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1271 previous_chunk = output_chunk;
1272 break;
1273 }
1274 }
16a4353c 1275 count_zero = fMaxMergeFiles;
c57f56b7 1276 while ((map=(TMap*)nextmap())) {
16a4353c 1277 // Loop 'find' results and get next LFN
1278 if (count_zero == fMaxMergeFiles) {
1279 // First file in chunk - create file merger and add previous chunk if any.
1280 fm = new TFileMerger(kFALSE);
1281 fm->SetFastMethod(kTRUE);
1282 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1283 output_chunk = output_file;
1284 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1285 }
1286 // If last file found, put merged results in the output file
1287 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1288 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1289 if (!objs || !objs->GetString().Length()) {
16a4353c 1290 // Nothing found - skip this output
c57f56b7 1291 delete res;
16a4353c 1292 delete fm;
1293 break;
1294 }
1295 // Add file to be merged and decrement chunk counter.
1296 fm->AddFile(objs->GetString());
1297 count_zero--;
1298 if (count_zero==0 || map == res->Last()) {
1299 fm->OutputFile(output_chunk);
1300 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1301 // Nothing found - skip this output
1302 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1303 delete res;
1304 delete fm;
1305 break;
1306 }
1307 // Merge the outputs, then go to next chunk
1308 if (!fm->Merge()) {
1309 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1310 delete res;
1311 delete fm;
1312 merged = kFALSE;
1313 break;
1314 } else {
1315 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1316 gSystem->Unlink(previous_chunk);
16a4353c 1317 }
1318 if (map == res->Last()) {
1319 delete res;
1320 delete fm;
1321 break;
1322 }
1323 count_chunk++;
1324 count_zero = fMaxMergeFiles;
1325 previous_chunk = output_chunk;
c57f56b7 1326 }
c57f56b7 1327 }
c57f56b7 1328 }
1329 if (!merged) {
1330 Error("MergeOutputs", "Terminate() will NOT be executed");
1331 }
1332 return merged;
1333}
1334
bb885a9e 1335//______________________________________________________________________________
1336void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1337{
1338// Use the output files connected to output containers from the analysis manager
1339// rather than the files defined by SetOutputFiles
1340 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1341 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1342 analysis manager");
1343 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1344}
1345
c57f56b7 1346//______________________________________________________________________________
5513444a 1347void AliAnalysisAlien::SetProductionMode(Bool_t flag)
1348{
1349// If production mode is set, all required files are produced and copied to
1350// AliEn but the master jobs are not submitted. A file .prod containing all
1351// submit parameters is copied to the work directory.
1352 if (flag && !TObject::TestBit(AliAnalysisGrid::kProductionMode))
1353 Info("SetProductionMode", "Plugin in production mode. Jobs are not submitted.");
1354 TObject::SetBit(AliAnalysisGrid::kProductionMode, flag);
1355}
1356
1357//______________________________________________________________________________
1358Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1359{
1360// Start remote grid analysis.
1361
43da816a 1362 // Check if output files have to be taken from the analysis manager
1363 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1364 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1365 if (!mgr || !mgr->IsInitialized()) {
1366 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1367 return kFALSE;
43da816a 1368 }
1369 fOutputFiles = "";
1370 TIter next(mgr->GetOutputs());
1371 AliAnalysisDataContainer *output;
1372 while ((output=(AliAnalysisDataContainer*)next())) {
1373 const char *filename = output->GetFileName();
1374 if (!(strcmp(filename, "default"))) {
1375 if (!mgr->GetOutputEventHandler()) continue;
1376 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1377 }
1378 if (fOutputFiles.Length()) fOutputFiles += " ";
1379 fOutputFiles += filename;
1380 }
c07b9ce2 1381 // Add extra files registered to the analysis manager
1382 if (mgr->GetExtraFiles().Length()) {
1383 if (fOutputFiles.Length()) fOutputFiles += " ";
1384 fOutputFiles += mgr->GetExtraFiles();
1385 }
43da816a 1386 }
f7b1cbc2 1387// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1388 if (TestBit(AliAnalysisGrid::kOffline)) {
1389 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1390 \n there nor any job run. You can revise the JDL and analysis \
1391 \n macro then run the same in \"submit\" mode.");
1392 } else if (TestBit(AliAnalysisGrid::kTest)) {
1393 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1394 \n dataset.");
1395 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1396 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1397 \n space and job submitted.");
1398 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1399 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1400 return kTRUE;
c57f56b7 1401 } else {
1402 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1403 }
1404
1405 if (!Connect()) {
1406 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1407 return kFALSE;
16a4353c 1408 }
1409 Print();
c57f56b7 1410 if (!CheckInputData()) {
1411 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1412 return kFALSE;
c57f56b7 1413 }
1414 CreateDataset(fDataPattern);
1415 WriteAnalysisFile();
1416 WriteAnalysisMacro();
1417 WriteExecutable();
1418 WriteValidationScript();
5513444a 1419 if (!CreateJDL()) return kFALSE;
1420 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1421 if (TestBit(AliAnalysisGrid::kTest)) {
1422 // Locally testing the analysis
1423 Info("StartAnalysis", "\n_______________________________________________________________________ \
1424 \n Running analysis script in a daughter shell as on a worker node \
1425 \n_______________________________________________________________________");
1426 TObjArray *list = fOutputFiles.Tokenize(" ");
1427 TIter next(list);
1428 TObjString *str;
1429 TString output_file;
1430 while((str=(TObjString*)next())) {
1431 output_file = str->GetString();
1432 Int_t index = output_file.Index("@");
1433 if (index > 0) output_file.Remove(index);
43da816a 1434 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1435 }
1436 delete list;
1437 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1438 gSystem->Exec("bash validate.sh");
1439// gSystem->Exec("cat stdout");
5513444a 1440 return kFALSE;
c57f56b7 1441 }
5513444a 1442 // Check if submitting is managed by LPM manager
1443 if (TObject::TestBit(AliAnalysisGrid::kProductionMode)) {
1444 TString prodfile = fJDLName;
1445 prodfile.ReplaceAll(".jdl", ".prod");
1446 WriteProductionFile(prodfile);
1447 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1448 return kFALSE;
1449 }
a8739e8a 1450 // Submit AliEn job(s)
d2a409b2 1451 gGrid->Cd(fGridOutputDir);
a8739e8a 1452 TGridResult *res;
c57f56b7 1453 TString jobID = "";
d2a409b2 1454 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1455 // Submit a given xml or a set of runs
a8739e8a 1456 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1457 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1458 if (res) {
1459 const char *cjobId = res->GetKey(0,"jobId");
1460 if (!cjobId) {
1461 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1462 return kFALSE;
a8739e8a 1463 } else {
1464 Info("StartAnalysis", "\n_______________________________________________________________________ \
1465 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1466 \n_______________________________________________________________________",
1467 fJDLName.Data(), cjobId);
1468 jobID = cjobId;
1469 }
1470 delete res;
1471 }
1472 } else {
d2a409b2 1473 // Submit for a range of enumeration of runs.
1474 Submit();
c57f56b7 1475 }
a8739e8a 1476
c57f56b7 1477 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1478 \n You may exit at any time and terminate the job later using the option <terminate> \
1479 \n ##################################################################################", jobID.Data());
bb885a9e 1480 gSystem->Exec("aliensh");
5513444a 1481 return kTRUE;
c57f56b7 1482}
1483
d2a409b2 1484//______________________________________________________________________________
1485void AliAnalysisAlien::Submit()
1486{
1487// Submit all master jobs.
1488 Int_t nmasterjobs = fInputFiles->GetEntries();
1489 Long_t tshoot = gSystem->Now();
1490 if (!fNsubmitted) SubmitNext();
1491 while (fNsubmitted < nmasterjobs) {
1492 Long_t now = gSystem->Now();
1493 if ((now-tshoot)>30000) {
1494 tshoot = now;
1495 SubmitNext();
1496 }
1497 }
1498}
1499
1500//______________________________________________________________________________
1501void AliAnalysisAlien::SubmitNext()
1502{
1503// Submit next bunch of master jobs if the queue is free.
1504 static Bool_t iscalled = kFALSE;
1505 static Int_t firstmaster = 0;
1506 static Int_t lastmaster = 0;
1507 static Int_t npermaster = 0;
1508 if (iscalled) return;
1509 iscalled = kTRUE;
1510 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1511 Int_t ntosubmit = 0;
1512 TGridResult *res;
1513 TString jobID = "";
1514 if (!fNsubmitted) ntosubmit = 1;
1515 else {
1516 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
1517 printf("=== master %d: %s\n", lastmaster, status.Data());
1518 // If last master not split, just return
1519 if (status != "SPLIT") {iscalled = kFALSE; return;}
1520 // No more than 100 waiting jobs
1521 if (nwaiting>100) {iscalled = kFALSE; return;}
1522 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1523 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
1524 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
1525 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1526 }
1527 Int_t nmasterjobs = fInputFiles->GetEntries();
1528 for (Int_t i=0; i<ntosubmit; i++) {
1529 // Submit for a range of enumeration of runs.
1530 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1531 TString query;
1532 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
1533 printf("********* %s\n",query.Data());
1534 res = gGrid->Command(query);
1535 if (res) {
98ca124f 1536 TString cjobId1 = res->GetKey(0,"jobId");
1537 if (!cjobId1.Length()) {
d2a409b2 1538 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1539 iscalled = kFALSE;
1540 return;
1541 } else {
1542 Info("StartAnalysis", "\n_______________________________________________________________________ \
1543 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1544 \n_______________________________________________________________________",
98ca124f 1545 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1546 jobID += cjobId1;
1547 jobID += " ";
98ca124f 1548 lastmaster = cjobId1.Atoi();
d2a409b2 1549 if (!firstmaster) firstmaster = lastmaster;
1550 fNsubmitted++;
1551 }
1552 delete res;
1553 }
1554 }
1555 iscalled = kFALSE;
1556}
1557
c57f56b7 1558//______________________________________________________________________________
1559void AliAnalysisAlien::WriteAnalysisFile()
1560{
1561// Write current analysis manager into the file analysis.root
1562 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1563 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1564 if (!mgr || !mgr->IsInitialized()) {
1565 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1566 return;
1567 }
1568 // Check analysis type
1569 TObject *handler;
1570 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1571 handler = (TObject*)mgr->GetInputEventHandler();
1572 if (handler) {
1573 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1574 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1575 }
1576 TDirectory *cdir = gDirectory;
1577 TFile *file = TFile::Open("analysis.root", "RECREATE");
1578 if (file) {
1579 mgr->Write();
1580 delete file;
1581 }
1582 if (cdir) cdir->cd();
1583 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1584 }
1585 Bool_t copy = kTRUE;
1586 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1587 if (copy) {
1588 CdWork();
1589 TString workdir = gGrid->GetHomeDirectory();
1590 workdir += fGridWorkingDir;
1591 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1592 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1593 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1594 }
1595}
1596
1597//______________________________________________________________________________
1598void AliAnalysisAlien::WriteAnalysisMacro()
1599{
1600// Write the analysis macro that will steer the analysis in grid mode.
1601 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1602 ofstream out;
1603 out.open(fAnalysisMacro.Data(), ios::out);
1604 if (!out.good()) {
1605 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1606 return;
1607 }
1608 TString func = fAnalysisMacro;
1609 TString type = "ESD";
1610 TString comment = "// Analysis using ";
1611 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1612 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1613 type = "AOD";
1614 comment += "AOD";
1615 }
0df6ccf2 1616 if (type!="AOD" && fFriendChainName!="") {
1617 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1618 return;
1619 }
c57f56b7 1620 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1621 else comment += " data";
1622 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1623 func.ReplaceAll(".C", "");
1624 out << "void " << func.Data() << "()" << endl;
1625 out << "{" << endl;
1626 out << comment.Data() << endl;
1627 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1628 out << " TStopwatch timer;" << endl;
1629 out << " timer.Start();" << endl << endl;
c57f56b7 1630 out << "// load base root libraries" << endl;
1631 out << " gSystem->Load(\"libTree\");" << endl;
1632 out << " gSystem->Load(\"libGeom\");" << endl;
1633 out << " gSystem->Load(\"libVMC\");" << endl;
1634 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1635 out << "// Load analysis framework libraries" << endl;
4e5c5506 1636 if (!fPackages) {
4e5c5506 1637 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1638 out << " gSystem->Load(\"libESD\");" << endl;
1639 out << " gSystem->Load(\"libAOD\");" << endl;
1640 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1641 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1642 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1643 } else {
4e5c5506 1644 TIter next(fPackages);
1645 TObject *obj;
57377eb5 1646 TString pkgname;
1647 Bool_t hasSTEERBase = kFALSE;
1648 Bool_t hasESD = kFALSE;
1649 Bool_t hasAOD = kFALSE;
1650 Bool_t hasANALYSIS = kFALSE;
1651 Bool_t hasANALYSISalice = kFALSE;
1652 Bool_t hasCORRFW = kFALSE;
1653 while ((obj=next())) {
1654 pkgname = obj->GetName();
4478e6f1 1655 if (pkgname == "STEERBase" ||
1656 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1657 if (pkgname == "ESD" ||
1658 pkgname == "ESD.par") hasESD = kTRUE;
1659 if (pkgname == "AOD" ||
1660 pkgname == "AOD.par") hasAOD = kTRUE;
1661 if (pkgname == "ANALYSIS" ||
1662 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1663 if (pkgname == "ANALYSISalice" ||
1664 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1665 if (pkgname == "CORRFW" ||
1666 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1667 }
1668 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1669 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1670 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1671 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1672 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1673 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1674 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1675 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1676 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1677 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1678 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1679 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1680 out << "// Compile other par packages" << endl;
1681 next.Reset();
fcc9bb6f 1682 while ((obj=next())) {
1683 pkgname = obj->GetName();
4478e6f1 1684 if (pkgname == "STEERBase" ||
1685 pkgname == "STEERBase.par" ||
1686 pkgname == "ESD" ||
1687 pkgname == "ESD.par" ||
1688 pkgname == "AOD" ||
1689 pkgname == "AOD.par" ||
1690 pkgname == "ANALYSIS" ||
1691 pkgname == "ANALYSIS.par" ||
1692 pkgname == "ANALYSISalice" ||
1693 pkgname == "ANALYSISalice.par" ||
1694 pkgname == "CORRFW" ||
1695 pkgname == "CORRFW.par") continue;
4e5c5506 1696 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1697 }
4e5c5506 1698 }
43da816a 1699 out << "// include path" << endl;
1700 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1701 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1702 if (fAdditionalLibs.Length()) {
1703 out << "// Add aditional AliRoot libraries" << endl;
1704 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1705 TIter next(list);
1706 TObjString *str;
1707 while((str=(TObjString*)next())) {
1708 if (str->GetString().Contains(".so"))
1709 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1710 }
1711 if (list) delete list;
1712 }
1713 out << endl;
c57f56b7 1714 out << "// analysis source to be compiled at runtime (if any)" << endl;
1715 if (fAnalysisSource.Length()) {
1716 TObjArray *list = fAnalysisSource.Tokenize(" ");
1717 TIter next(list);
1718 TObjString *str;
1719 while((str=(TObjString*)next())) {
1720 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1721 }
1722 if (list) delete list;
1723 }
1724 out << endl;
1725 out << "// connect to AliEn and make the chain" << endl;
1726 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1727 if (IsUsingTags()) {
1728 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1729 } else {
1730 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1731 }
1732 out << "// read the analysis manager from file" << endl;
1733 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1734 out << " if (!file) return;" << endl;
1735 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1736 out << " AliAnalysisManager *mgr = 0;" << endl;
1737 out << " TKey *key;" << endl;
1738 out << " while ((key=(TKey*)nextkey())) {" << endl;
1739 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1740 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1741 out << " };" << endl;
1742 out << " if (!mgr) {" << endl;
1743 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1744 out << " return;" << endl;
1745 out << " }" << endl << endl;
1746 out << " mgr->PrintStatus();" << endl;
1747 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1748 out << " timer.Stop();" << endl;
1749 out << " timer.Print();" << endl;
c57f56b7 1750 out << "}" << endl << endl;
1751 if (IsUsingTags()) {
1752 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1753 out << "{" << endl;
1754 out << "// Create a chain using tags from the xml file." << endl;
1755 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1756 out << " if (!coll) {" << endl;
1757 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1758 out << " return NULL;" << endl;
1759 out << " }" << endl;
1760 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1761 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1762 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1763 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1764 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1765 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1766 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1767 out << " // Check if the cuts configuration file was provided" << endl;
1768 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1769 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1770 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1771 out << " }" << endl;
0df6ccf2 1772 if (fFriendChainName=="") {
1773 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1774 } else {
1775 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1776 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1777 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1778 }
c57f56b7 1779 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1780 out << " chain->ls();" << endl;
1781 out << " return chain;" << endl;
fcc9bb6f 1782 out << "}" << endl << endl;
c57f56b7 1783 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1784 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1785 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1786 msg += " AliLHCTagCuts *lhcCuts,\n";
1787 msg += " AliDetectorTagCuts *detCuts,\n";
1788 msg += " AliEventTagCuts *evCuts)";
1789 Info("WriteAnalysisMacro", msg.Data());
1790 }
0df6ccf2 1791 }
1792 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1793 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1794 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1795 out << "{" << endl;
1796 out << "// Create a chain using url's from xml file" << endl;
1797 out << " TString treename = type;" << endl;
1798 out << " treename.ToLower();" << endl;
1799 out << " treename += \"Tree\";" << endl;
e02fee64 1800 out << " printf(\"***************************************\\n\");" << endl;
1801 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1802 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1803 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1804 out << " if (!coll) {" << endl;
1805 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1806 out << " return NULL;" << endl;
1807 out << " }" << endl;
1808 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1809 if(fFriendChainName!="") {
1810 out << " TChain *chainFriend = new TChain(treename);" << endl;
1811 }
c57f56b7 1812 out << " coll->Reset();" << endl;
0df6ccf2 1813 out << " while (coll->Next()) {" << endl;
1814 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1815 if(fFriendChainName!="") {
1816 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1817 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1818 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1819 out << " chainFriend->Add(fileFriend.Data());" << endl;
1820 }
1821 out << " }" << endl;
c57f56b7 1822 out << " if (!chain->GetNtrees()) {" << endl;
1823 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1824 out << " return NULL;" << endl;
1825 out << " }" << endl;
0df6ccf2 1826 if(fFriendChainName!="") {
1827 out << " chain->AddFriend(chainFriend);" << endl;
1828 }
c57f56b7 1829 out << " return chain;" << endl;
fcc9bb6f 1830 out << "}" << endl << endl;
c57f56b7 1831 }
4e5c5506 1832 if (fPackages) {
fcc9bb6f 1833 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1834 out << "Bool_t SetupPar(const char *package) {" << endl;
1835 out << "// Compile the package and set it up." << endl;
1836 out << " TString pkgdir = package;" << endl;
1837 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1838 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1839 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1840 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1841 out << " // Check for BUILD.sh and execute" << endl;
1842 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1843 out << " printf(\"*******************************\\n\");" << endl;
1844 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1845 out << " printf(\"*******************************\\n\");" << endl;
1846 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1847 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1848 out << " gSystem->ChangeDirectory(cdir);" << endl;
1849 out << " return kFALSE;" << endl;
1850 out << " }" << endl;
1851 out << " } else {" << endl;
fcc9bb6f 1852 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1853 out << " gSystem->ChangeDirectory(cdir);" << endl;
1854 out << " return kFALSE;" << endl;
1855 out << " }" << endl;
1856 out << " // Check for SETUP.C and execute" << endl;
1857 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1858 out << " printf(\"*******************************\\n\");" << endl;
1859 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1860 out << " printf(\"*******************************\\n\");" << endl;
1861 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1862 out << " } else {" << endl;
fcc9bb6f 1863 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1864 out << " gSystem->ChangeDirectory(cdir);" << endl;
1865 out << " return kFALSE;" << endl;
1866 out << " }" << endl;
1867 out << " // Restore original workdir" << endl;
1868 out << " gSystem->ChangeDirectory(cdir);" << endl;
1869 out << " return kTRUE;" << endl;
1870 out << "}" << endl;
1871 }
c57f56b7 1872 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1873 }
1874 Bool_t copy = kTRUE;
1875 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1876 if (copy) {
1877 CdWork();
1878 TString workdir = gGrid->GetHomeDirectory();
1879 workdir += fGridWorkingDir;
1880 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1881 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1882 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1883 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1884 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1885 }
1886 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1887 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1888 }
1889}
1890
1891//______________________________________________________________________________
1892void AliAnalysisAlien::WriteExecutable()
1893{
1894// Generate the alien executable script.
1895 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1896 ofstream out;
1897 out.open(fExecutable.Data(), ios::out);
1898 if (out.bad()) {
5513444a 1899 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1900 return;
1901 }
1902 out << "#!/bin/bash" << endl;
1903 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1904 out << "echo \"=========================================\"" << endl;
1905 out << "echo \"############## PATH : ##############\"" << endl;
1906 out << "echo $PATH" << endl;
1907 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1908 out << "echo $LD_LIBRARY_PATH" << endl;
1909 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1910 out << "echo $ROOTSYS" << endl;
1911 out << "echo \"############## which root : ##############\"" << endl;
1912 out << "which root" << endl;
1913 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1914 out << "echo $ALICE_ROOT" << endl;
1915 out << "echo \"############## which aliroot : ##############\"" << endl;
1916 out << "which aliroot" << endl;
1917 out << "echo \"=========================================\"" << endl << endl;
1918// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1919 out << "root -b -q ";
1920 out << fAnalysisMacro.Data() << endl << endl;
1921 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1922 }
1923 Bool_t copy = kTRUE;
1924 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1925 if (copy) {
1926 CdWork();
1927 TString workdir = gGrid->GetHomeDirectory();
1928 workdir += fGridWorkingDir;
1929 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1930 if (FileExists(executable)) gGrid->Rm(executable);
1931 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1932 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1933 }
1934}
1935
1936//______________________________________________________________________________
5513444a 1937void AliAnalysisAlien::WriteProductionFile(const char *filename) const
1938{
1939// Write the production file to be submitted by LPM manager. The format is:
1940// First line: full_path_to_jdl
1941// Next lines: full_path_to_dataset XXX (XXX is a string)
1942// To submit, one has to: submit jdl XXX for all lines
1943 ofstream out;
1944 out.open(filename, ios::out);
1945 if (out.bad()) {
1946 Error("WriteProductionFile", "Bad file name: %s", filename);
1947 return;
1948 }
1949 TString workdir = gGrid->GetHomeDirectory();
1950 workdir += fGridWorkingDir;
1951 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
1952 out << locjdl << endl;
1953 Int_t nmasterjobs = fInputFiles->GetEntries();
1954 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 1955 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 1956 }
1957 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
1958 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
1959}
1960
1961//______________________________________________________________________________
c57f56b7 1962void AliAnalysisAlien::WriteValidationScript()
1963{
1964// Generate the alien validation script.
1965 // Generate the validation script
1966 TObjString *os;
1967 if (!Connect()) {
1968 Error("WriteValidationScript", "Alien connection required");
1969 return;
1970 }
1971 TString out_stream = "";
1972 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1973 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1974 ofstream out;
1975 out.open("validate.sh", ios::out);
1976 out << "#!/bin/bash" << endl;
1977 out << "##################################################" << endl;
1978 out << "validateout=`dirname $0`" << endl;
1979 out << "validatetime=`date`" << endl;
1980 out << "validated=\"0\";" << endl;
1981 out << "error=0" << endl;
1982 out << "if [ -z $validateout ]" << endl;
1983 out << "then" << endl;
1984 out << " validateout=\".\"" << endl;
1985 out << "fi" << endl << endl;
1986 out << "cd $validateout;" << endl;
1987 out << "validateworkdir=`pwd`;" << endl << endl;
1988 out << "echo \"*******************************************************\"" << out_stream << endl;
1989 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1990 out << "" << endl;
1991 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1992 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1993 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1994 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1995 out << "ls -la ./" << out_stream << endl;
1996 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1997 out << "##################################################" << endl;
ebec370a 1998
1999 out << "" << endl;
2000 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2001 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2002 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2003 out << "" << endl;
2004
2005 out << "if [ ! -f stderr ] ; then" << endl;
2006 out << " error=1" << endl;
2007 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2008 out << " echo \"Error = $error\" " << out_stream << endl;
2009 out << "fi" << endl;
2010
2011 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2012 out << " error=1" << endl;
2013 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2014 out << " echo \"$parArch\" " << out_stream << endl;
2015 out << " echo \"Error = $error\" " << out_stream << endl;
2016 out << "fi" << endl;
2017
2018 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2019 out << " error=1" << endl;
2020 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2021 out << " echo \"$segViol\" " << out_stream << endl;
2022 out << " echo \"Error = $error\" " << out_stream << endl;
2023 out << "fi" << endl;
2024
2025 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2026 out << " error=1" << endl;
2027 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2028 out << " echo \"$segFault\" " << out_stream << endl;
2029 out << " echo \"Error = $error\" " << out_stream << endl;
2030 out << "fi" << endl;
2031
2032 // Part dedicated to the specific analyses running into the train
2033
c57f56b7 2034 TObjArray *arr = fOutputFiles.Tokenize(" ");
2035 TIter next1(arr);
2036 TString output_file;
2037 while ((os=(TObjString*)next1())) {
2038 output_file = os->GetString();
2039 Int_t index = output_file.Index("@");
2040 if (index > 0) output_file.Remove(index);
2041 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2042 out << " error=1" << endl;
2043 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2044 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2045 out << "fi" << endl;
2046 }
2047 delete arr;
2048 out << "if [ $error = 0 ] ; then" << endl;
2049 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2050 out << "fi" << endl;
2051
2052 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2053 out << "echo \"*******************************************************\"" << out_stream << endl;
2054 out << "cd -" << endl;
2055 out << "exit $error" << endl;
2056 }
2057 Bool_t copy = kTRUE;
2058 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2059 if (copy) {
2060 CdWork();
2061 TString workdir = gGrid->GetHomeDirectory();
2062 workdir += fGridWorkingDir;
2063 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
2064 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
2065 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
2066 }
2067}