]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix coding violations
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
59 fArguments(),
60 fAnalysisMacro(),
61 fAnalysisSource(),
62 fAdditionalLibs(),
63 fSplitMode(),
64 fAPIVersion(),
65 fROOTVersion(),
66 fAliROOTVersion(),
67 fUser(),
68 fGridWorkingDir(),
69 fGridDataDir(),
70 fDataPattern(),
71 fGridOutputDir(),
72 fOutputArchive(),
73 fOutputFiles(),
74 fInputFormat(),
e7c71df0 75 fDatasetName(),
c57f56b7 76 fJDLName(),
bb885a9e 77 fMergeExcludes(),
f965131e 78 fIncludePath(),
bb885a9e 79 fCloseSE(),
0df6ccf2 80 fFriendChainName(),
c6cb3634 81 fJobTag(),
4e5c5506 82 fInputFiles(0),
83 fPackages(0)
c57f56b7 84{
85// Dummy ctor.
86 SetDefaults();
87}
88
89//______________________________________________________________________________
90AliAnalysisAlien::AliAnalysisAlien(const char *name)
91 :AliAnalysisGrid(name),
92 fGridJDL(NULL),
93 fPrice(0),
94 fTTL(0),
95 fSplitMaxInputFileNumber(0),
96 fMaxInitFailed(0),
97 fMasterResubmitThreshold(0),
bb885a9e 98 fNtestFiles(0),
319593fb 99 fNrunsPerMaster(0),
16a4353c 100 fMaxMergeFiles(0),
d2a409b2 101 fNsubmitted(0),
a3e84053 102 fProductionMode(0),
c57f56b7 103 fRunNumbers(),
104 fExecutable(),
105 fArguments(),
106 fAnalysisMacro(),
107 fAnalysisSource(),
108 fAdditionalLibs(),
109 fSplitMode(),
110 fAPIVersion(),
111 fROOTVersion(),
112 fAliROOTVersion(),
113 fUser(),
114 fGridWorkingDir(),
115 fGridDataDir(),
116 fDataPattern(),
117 fGridOutputDir(),
118 fOutputArchive(),
119 fOutputFiles(),
120 fInputFormat(),
e7c71df0 121 fDatasetName(),
c57f56b7 122 fJDLName(),
bb885a9e 123 fMergeExcludes(),
f965131e 124 fIncludePath(),
bb885a9e 125 fCloseSE(),
0df6ccf2 126 fFriendChainName(),
c6cb3634 127 fJobTag(),
4e5c5506 128 fInputFiles(0),
129 fPackages(0)
c57f56b7 130{
131// Default ctor.
132 SetDefaults();
133}
134
135//______________________________________________________________________________
136AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
137 :AliAnalysisGrid(other),
138 fGridJDL(NULL),
139 fPrice(other.fPrice),
140 fTTL(other.fTTL),
141 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
142 fMaxInitFailed(other.fMaxInitFailed),
143 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 144 fNtestFiles(other.fNtestFiles),
319593fb 145 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 146 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 147 fNsubmitted(other.fNsubmitted),
a3e84053 148 fProductionMode(other.fProductionMode),
c57f56b7 149 fRunNumbers(other.fRunNumbers),
150 fExecutable(other.fExecutable),
151 fArguments(other.fArguments),
152 fAnalysisMacro(other.fAnalysisMacro),
153 fAnalysisSource(other.fAnalysisSource),
154 fAdditionalLibs(other.fAdditionalLibs),
155 fSplitMode(other.fSplitMode),
156 fAPIVersion(other.fAPIVersion),
157 fROOTVersion(other.fROOTVersion),
158 fAliROOTVersion(other.fAliROOTVersion),
159 fUser(other.fUser),
160 fGridWorkingDir(other.fGridWorkingDir),
161 fGridDataDir(other.fGridDataDir),
162 fDataPattern(other.fDataPattern),
163 fGridOutputDir(other.fGridOutputDir),
164 fOutputArchive(other.fOutputArchive),
165 fOutputFiles(other.fOutputFiles),
166 fInputFormat(other.fInputFormat),
e7c71df0 167 fDatasetName(other.fDatasetName),
c57f56b7 168 fJDLName(other.fJDLName),
bb885a9e 169 fMergeExcludes(other.fMergeExcludes),
f965131e 170 fIncludePath(other.fIncludePath),
bb885a9e 171 fCloseSE(other.fCloseSE),
0df6ccf2 172 fFriendChainName(other.fFriendChainName),
c6cb3634 173 fJobTag(other.fJobTag),
4e5c5506 174 fInputFiles(0),
175 fPackages(0)
c57f56b7 176{
177// Copy ctor.
178 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 179 fRunRange[0] = other.fRunRange[0];
180 fRunRange[1] = other.fRunRange[1];
c57f56b7 181 if (other.fInputFiles) {
182 fInputFiles = new TObjArray();
183 TIter next(other.fInputFiles);
184 TObject *obj;
185 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
186 fInputFiles->SetOwner();
187 }
4e5c5506 188 if (other.fPackages) {
189 fPackages = new TObjArray();
190 TIter next(other.fPackages);
191 TObject *obj;
192 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
193 fPackages->SetOwner();
194 }
c57f56b7 195}
196
197//______________________________________________________________________________
198AliAnalysisAlien::~AliAnalysisAlien()
199{
200// Destructor.
201 if (fGridJDL) delete fGridJDL;
202 if (fInputFiles) delete fInputFiles;
4e5c5506 203 if (fPackages) delete fPackages;
c57f56b7 204}
205
206//______________________________________________________________________________
207AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
208{
209// Assignment.
210 if (this != &other) {
211 AliAnalysisGrid::operator=(other);
212 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
213 fPrice = other.fPrice;
214 fTTL = other.fTTL;
215 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
216 fMaxInitFailed = other.fMaxInitFailed;
217 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 218 fNtestFiles = other.fNtestFiles;
a3e84053 219 fNrunsPerMaster = other.fNrunsPerMaster;
220 fMaxMergeFiles = other.fMaxMergeFiles;
221 fNsubmitted = other.fNsubmitted;
222 fProductionMode = other.fProductionMode;
c57f56b7 223 fRunNumbers = other.fRunNumbers;
224 fExecutable = other.fExecutable;
225 fArguments = other.fArguments;
226 fAnalysisMacro = other.fAnalysisMacro;
227 fAnalysisSource = other.fAnalysisSource;
228 fAdditionalLibs = other.fAdditionalLibs;
229 fSplitMode = other.fSplitMode;
230 fAPIVersion = other.fAPIVersion;
231 fROOTVersion = other.fROOTVersion;
232 fAliROOTVersion = other.fAliROOTVersion;
233 fUser = other.fUser;
234 fGridWorkingDir = other.fGridWorkingDir;
235 fGridDataDir = other.fGridDataDir;
236 fDataPattern = other.fDataPattern;
237 fGridOutputDir = other.fGridOutputDir;
238 fOutputArchive = other.fOutputArchive;
239 fOutputFiles = other.fOutputFiles;
240 fInputFormat = other.fInputFormat;
e7c71df0 241 fDatasetName = other.fDatasetName;
c57f56b7 242 fJDLName = other.fJDLName;
bb885a9e 243 fMergeExcludes = other.fMergeExcludes;
f965131e 244 fIncludePath = other.fIncludePath;
bb885a9e 245 fCloseSE = other.fCloseSE;
0df6ccf2 246 fFriendChainName = other.fFriendChainName;
c6cb3634 247 fJobTag = other.fJobTag;
c57f56b7 248 if (other.fInputFiles) {
249 fInputFiles = new TObjArray();
250 TIter next(other.fInputFiles);
251 TObject *obj;
252 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
253 fInputFiles->SetOwner();
254 }
4e5c5506 255 if (other.fPackages) {
256 fPackages = new TObjArray();
257 TIter next(other.fPackages);
258 TObject *obj;
259 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
260 fPackages->SetOwner();
261 }
c57f56b7 262 }
263 return *this;
264}
265
f965131e 266//______________________________________________________________________________
267void AliAnalysisAlien::AddIncludePath(const char *path)
268{
269// Add include path in the remote analysis macro.
270 TString p(path);
271 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
272 else fIncludePath += Form("-I%s ", path);
273}
274
c57f56b7 275//______________________________________________________________________________
276void AliAnalysisAlien::AddRunNumber(Int_t run)
277{
278// Add a run number to the list of runs to be processed.
279 if (fRunNumbers.Length()) fRunNumbers += " ";
280 fRunNumbers += Form("%d", run);
281}
282
ee75cfc3 283//______________________________________________________________________________
284void AliAnalysisAlien::AddRunNumber(const char* run)
285{
286// Add a run number to the list of runs to be processed.
287 if (fRunNumbers.Length()) fRunNumbers += " ";
288 fRunNumbers += run;
289}
290
c57f56b7 291//______________________________________________________________________________
292void AliAnalysisAlien::AddDataFile(const char *lfn)
293{
294// Adds a data file to the input to be analysed. The file should be a valid LFN
295// or point to an existing file in the alien workdir.
296 if (!fInputFiles) fInputFiles = new TObjArray();
297 fInputFiles->Add(new TObjString(lfn));
298}
299
300//______________________________________________________________________________
301Bool_t AliAnalysisAlien::Connect()
302{
303// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
304 if (gGrid && gGrid->IsConnected()) return kTRUE;
305 if (!gSystem->Getenv("alien_API_USER")) {
306 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
307 gSystem->Getenv("UID"));
308 return kFALSE;
309 }
310 if (!gGrid) {
311 Info("Connect", "Trying to connect to AliEn ...");
312 TGrid::Connect("alien://");
313 }
314 if (!gGrid || !gGrid->IsConnected()) {
315 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
316 return kFALSE;
317 }
318 fUser = gGrid->GetUser();
319 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
320 return kTRUE;
321}
322
323//______________________________________________________________________________
324void AliAnalysisAlien::CdWork()
325{
326// Check validity of alien workspace. Create directory if possible.
327 if (!Connect()) {
328 Error("CdWork", "Alien connection required");
329 return;
330 }
331 TString homedir = gGrid->GetHomeDirectory();
332 TString workdir = homedir + fGridWorkingDir;
923e2ca5 333 if (DirectoryExists(workdir)) {
334 gGrid->Cd(workdir);
335 return;
336 }
337 // Work directory not existing - create it
338 gGrid->Cd(homedir);
339 if (gGrid->Mkdir(workdir)) {
340 gGrid->Cd(fGridWorkingDir);
341 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
342 } else {
343 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
344 workdir.Data(), homedir.Data());
345 fGridWorkingDir = "";
346 }
c57f56b7 347}
348
349//______________________________________________________________________________
350Bool_t AliAnalysisAlien::CheckInputData()
351{
352// Check validity of input data. If necessary, create xml files.
a8739e8a 353 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 354 if (!fGridDataDir.Length()) {
355 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
356 return kFALSE;
357 }
358 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
359 return kTRUE;
c57f56b7 360 }
361 // Process declared files
362 Bool_t is_collection = kFALSE;
363 Bool_t is_xml = kFALSE;
364 Bool_t use_tags = kFALSE;
365 Bool_t checked = kFALSE;
366 CdWork();
367 TString file;
368 TString workdir = gGrid->GetHomeDirectory();
369 workdir += fGridWorkingDir;
370 if (fInputFiles) {
371 TObjString *objstr;
372 TIter next(fInputFiles);
373 while ((objstr=(TObjString*)next())) {
374 file = workdir;
375 file += "/";
376 file += objstr->GetString();
377 // Store full lfn path
378 if (FileExists(file)) objstr->SetString(file);
379 else {
380 file = objstr->GetName();
381 if (!FileExists(objstr->GetName())) {
382 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
383 objstr->GetName(), workdir.Data());
384 return kFALSE;
385 }
386 }
387 Bool_t iscoll, isxml, usetags;
388 CheckDataType(file, iscoll, isxml, usetags);
389 if (!checked) {
390 checked = kTRUE;
391 is_collection = iscoll;
392 is_xml = isxml;
393 use_tags = usetags;
394 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
395 } else {
396 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
397 Error("CheckInputData", "Some conflict was found in the types of inputs");
398 return kFALSE;
399 }
400 }
401 }
402 }
403 // Process requested run numbers
a8739e8a 404 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 405 // Check validity of alien data directory
406 if (!fGridDataDir.Length()) {
407 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
408 return kFALSE;
409 }
923e2ca5 410 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 411 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
412 return kFALSE;
413 }
414 if (is_collection) {
415 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
416 return kFALSE;
417 }
418
419 if (checked && !is_xml) {
420 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
421 return kFALSE;
422 }
423 // Check validity of run number(s)
424 TObjArray *arr;
425 TObjString *os;
319593fb 426 Int_t nruns = 0;
427 TString schunk;
c57f56b7 428 TString path;
429 if (!checked) {
430 checked = kTRUE;
431 use_tags = fDataPattern.Contains("tag");
432 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
433 }
434 if (use_tags != fDataPattern.Contains("tag")) {
435 Error("CheckInputData", "Cannot mix input files using/not using tags");
436 return kFALSE;
437 }
438 if (fRunNumbers.Length()) {
a8739e8a 439 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 440 arr = fRunNumbers.Tokenize(" ");
441 TIter next(arr);
442 while ((os=(TObjString*)next())) {
443 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 444 if (!DirectoryExists(path)) {
445 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 446 continue;
c57f56b7 447 }
448 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
449 TString msg = "\n##### file: ";
450 msg += path;
451 msg += " type: xml_collection;";
452 if (use_tags) msg += " using_tags: Yes";
453 else msg += " using_tags: No";
454 Info("CheckDataType", msg.Data());
319593fb 455 if (fNrunsPerMaster<2) {
d2a409b2 456 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 457 } else {
458 nruns++;
459 if (((nruns-1)%fNrunsPerMaster) == 0) {
460 schunk = os->GetString();
461 }
462 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
463 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 464 AddDataFile(schunk);
319593fb 465 }
c57f56b7 466 }
467 delete arr;
a8739e8a 468 } else {
469 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
470 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
471 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 472 if (!DirectoryExists(path)) {
473// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 474 continue;
475 }
476 path = Form("%s/%d.xml", workdir.Data(),irun);
477 TString msg = "\n##### file: ";
478 msg += path;
479 msg += " type: xml_collection;";
480 if (use_tags) msg += " using_tags: Yes";
481 else msg += " using_tags: No";
482 Info("CheckDataType", msg.Data());
319593fb 483 if (fNrunsPerMaster<2) {
d2a409b2 484 AddDataFile(Form("%d.xml",irun));
319593fb 485 } else {
486 nruns++;
487 if (((nruns-1)%fNrunsPerMaster) == 0) {
488 schunk = Form("%d", irun);
489 }
490 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
491 schunk += Form("_%d.xml", irun);
d2a409b2 492 AddDataFile(schunk);
319593fb 493 }
a8739e8a 494 }
c57f56b7 495 }
496 return kTRUE;
497}
498
499//______________________________________________________________________________
500Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
501{
502// Create dataset for the grid data directory + run number.
503 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
504 if (!Connect()) {
505 Error("CreateDataset", "Cannot create dataset with no grid connection");
506 return kFALSE;
507 }
508
509 // Cd workspace
510 CdWork();
511 TString workdir = gGrid->GetHomeDirectory();
512 workdir += fGridWorkingDir;
513
514 // Compose the 'find' command arguments
515 TString command;
516 TString options = "-x collection ";
bb885a9e 517 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 518 TString conditions = "";
519
520 TString file;
521 TString path;
319593fb 522 Int_t nruns = 0;
523 TString schunk;
ab254fd1 524 TGridCollection *cbase=0, *cadd=0;
d2a409b2 525 if (!fRunNumbers.Length() && !fRunRange[0]) {
526 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
527 // Make a single data collection from data directory.
528 path = fGridDataDir;
923e2ca5 529 if (!DirectoryExists(path)) {
d2a409b2 530 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
531 return kFALSE;
532 }
923e2ca5 533// CdWork();
d2a409b2 534 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
535 else file = Form("%s.xml", gSystem->BaseName(path));
536 if (gSystem->AccessPathName(file)) {
537 command = "find ";
538 command += options;
539 command += path;
540 command += " ";
541 command += pattern;
542 command += conditions;
923e2ca5 543 Printf("command: %s", command.Data());
d2a409b2 544 TGridResult *res = gGrid->Command(command);
545 if (res) delete res;
546 // Write standard output to file
547 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
548 }
549 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
550 // Copy xml file to alien space
551 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
552 if (!FileExists(file)) {
553 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
554 return kFALSE;
555 }
556 // Update list of files to be processed.
557 }
558 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
559 return kTRUE;
560 }
c57f56b7 561 // Several runs
a8739e8a 562 if (fRunNumbers.Length()) {
563 TObjArray *arr = fRunNumbers.Tokenize(" ");
564 TObjString *os;
565 TIter next(arr);
566 while ((os=(TObjString*)next())) {
567 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 568 if (!DirectoryExists(path)) continue;
569// CdWork();
a8739e8a 570 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
571 else file = Form("%s.xml", os->GetString().Data());
319593fb 572 // If local collection file does not exist, create it via 'find' command.
573 if (gSystem->AccessPathName(file)) {
574 command = "find ";
575 command += options;
576 command += path;
577 command += pattern;
578 command += conditions;
579 TGridResult *res = gGrid->Command(command);
580 if (res) delete res;
581 // Write standard output to file
582 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
583 }
a8739e8a 584 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 585 // Check if there is one run per master job.
586 if (fNrunsPerMaster<2) {
587 if (FileExists(file)) {
588 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
589 continue;
590 }
591 // Copy xml file to alien space
592 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
593 if (!FileExists(file)) {
594 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
595 delete arr;
596 return kFALSE;
597 }
598 } else {
599 nruns++;
600 if (((nruns-1)%fNrunsPerMaster) == 0) {
601 schunk = os->GetString();
602 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
603 } else {
604 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
923e2ca5 605 Printf(" Merging collection <%s> into masterjob input...", file.Data());
319593fb 606 cbase->Add(cadd);
607 delete cadd;
608 }
609 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
610 continue;
611 }
612 schunk += Form("_%s.xml", os->GetString().Data());
613 if (FileExists(schunk)) {
614 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
615 continue;
616 }
923e2ca5 617 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 618 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
619 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
620 if (!FileExists(schunk)) {
621 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
622 delete arr;
623 return kFALSE;
624 }
625 }
a8739e8a 626 }
627 delete arr;
628 } else {
629 // Process a full run range.
630 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
631 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 632 if (!DirectoryExists(path)) continue;
633// CdWork();
a8739e8a 634 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
635 else file = Form("%d.xml", irun);
319593fb 636 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 637 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
638// gGrid->Rm(file);
639 continue;
640 }
319593fb 641 // If local collection file does not exist, create it via 'find' command.
642 if (gSystem->AccessPathName(file)) {
643 command = "find ";
644 command += options;
645 command += path;
646 command += pattern;
647 command += conditions;
648 TGridResult *res = gGrid->Command(command);
649 if (res) delete res;
650 // Write standard output to file
651 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
652 }
a8739e8a 653 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 654 // Check if there is one run per master job.
655 if (fNrunsPerMaster<2) {
656 if (FileExists(file)) {
657 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
658 continue;
659 }
660 // Copy xml file to alien space
661 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
662 if (!FileExists(file)) {
663 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
664 return kFALSE;
665 }
666 } else {
667 nruns++;
95e5b448 668 // Check if the collection for the chunk exist locally.
669 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
670 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
923e2ca5 671 Printf(" Merging collection <%s> into %d runs chunk...",file.Data(),fNrunsPerMaster);
319593fb 672 if (((nruns-1)%fNrunsPerMaster) == 0) {
673 schunk = Form("%d", irun);
674 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
675 } else {
676 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
677 cbase->Add(cadd);
678 delete cadd;
679 }
680 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
681 continue;
682 }
683 schunk += Form("_%d.xml", irun);
684 if (FileExists(schunk)) {
685 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
686 continue;
687 }
923e2ca5 688 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 689 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 690 if (FileExists(schunk)) {
691 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
692 continue;
693 }
319593fb 694 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
695 if (!FileExists(schunk)) {
696 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
697 return kFALSE;
698 }
699 }
c57f56b7 700 }
a8739e8a 701 }
c57f56b7 702 return kTRUE;
703}
704
705//______________________________________________________________________________
706Bool_t AliAnalysisAlien::CreateJDL()
707{
708// Generate a JDL file according to current settings. The name of the file is
709// specified by fJDLName.
710 Bool_t error = kFALSE;
711 TObjArray *arr = 0;
712 Bool_t copy = kTRUE;
713 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
714 Bool_t generate = kTRUE;
715 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
716 if (!Connect()) {
717 Error("CreateJDL", "Alien connection required");
718 return kFALSE;
719 }
720 // Check validity of alien workspace
721 CdWork();
722 TString workdir = gGrid->GetHomeDirectory();
723 workdir += fGridWorkingDir;
724 if (generate) {
725 TObjString *os;
726 if (!fInputFiles) {
727 Error("CreateJDL()", "Define some input files for your analysis.");
728 error = kTRUE;
729 }
730 // Compose list of input files
731 // Check if output files were defined
732 if (!fOutputFiles.Length()) {
733 Error("CreateJDL", "You must define at least one output file");
734 error = kTRUE;
735 }
736 // Check if an output directory was defined and valid
737 if (!fGridOutputDir.Length()) {
738 Error("CreateJDL", "You must define AliEn output directory");
739 error = kTRUE;
740 } else {
d2a409b2 741 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 742 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 743 if (gGrid->Mkdir(fGridOutputDir)) {
744 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
745 } else {
746 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 747 // error = kTRUE;
c57f56b7 748 }
749 }
750 gGrid->Cd(workdir);
751 }
752 // Exit if any error up to now
753 if (error) return kFALSE;
754 // Set JDL fields
755 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 756 fGridJDL->SetExecutable(fExecutable);
c57f56b7 757// fGridJDL->SetTTL((UInt_t)fTTL);
758 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
759 if (fMaxInitFailed > 0)
760 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
761 if (fSplitMaxInputFileNumber > 0)
762 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
763 if (fSplitMode.Length())
764 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
765// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
766 if (fAliROOTVersion.Length())
767 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
768 if (fROOTVersion.Length())
769 fGridJDL->AddToPackages("ROOT", fROOTVersion);
770 if (fAPIVersion.Length())
771 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
772 fGridJDL->SetInputDataListFormat(fInputFormat);
773 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 774 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 775 TString analysisFile = fExecutable;
776 analysisFile.ReplaceAll(".sh", ".root");
777 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 778 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
779 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
780 if (fAdditionalLibs.Length()) {
781 arr = fAdditionalLibs.Tokenize(" ");
782 TIter next(arr);
783 while ((os=(TObjString*)next())) {
784 if (os->GetString().Contains(".so")) continue;
785 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
786 }
787 delete arr;
788 }
4e5c5506 789 if (fPackages) {
790 TIter next(fPackages);
791 TObject *obj;
792 while ((obj=next()))
793 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
794 }
c57f56b7 795 if (fOutputArchive.Length()) {
796 arr = fOutputArchive.Tokenize(" ");
797 TIter next(arr);
798 while ((os=(TObjString*)next()))
43da816a 799 if (!os->GetString().Contains("@") && fCloseSE.Length())
800 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
801 else
802 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 803 delete arr;
804 }
c57f56b7 805 arr = fOutputFiles.Tokenize(" ");
806 TIter next(arr);
43da816a 807 while ((os=(TObjString*)next())) {
e1eaf596 808 // Ignore ouputs in jdl that are also in outputarchive
809 TString sout = os->GetString();
810 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
811 if (fOutputArchive.Contains(sout)) continue;
43da816a 812 if (!os->GetString().Contains("@") && fCloseSE.Length())
813 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
814 else
815 fGridJDL->AddToOutputSandbox(os->GetString());
816 }
c57f56b7 817 delete arr;
818// fGridJDL->SetPrice((UInt_t)fPrice);
819 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 820 TString validationScript = fExecutable;
821 validationScript.ReplaceAll(".sh", "_validation.sh");
822 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 823 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 824 // Write a jdl with 2 input parameters: collection name and output dir name.
825 WriteJDL(copy);
c57f56b7 826 }
827 // Copy jdl to grid workspace
a8739e8a 828 if (copy) {
c57f56b7 829 if (fAdditionalLibs.Length()) {
e7c71df0 830 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 831 TObjString *os;
832 TIter next(arr);
833 while ((os=(TObjString*)next())) {
c57f56b7 834 if (os->GetString().Contains(".so")) continue;
4e5c5506 835 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 836 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
837 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
838 }
839 delete arr;
840 }
4e5c5506 841 if (fPackages) {
842 TIter next(fPackages);
843 TObject *obj;
844 while ((obj=next())) {
845 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
846 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
847 }
848 }
c57f56b7 849 }
850 return kTRUE;
851}
852
a8739e8a 853//______________________________________________________________________________
d2a409b2 854Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 855{
856// Writes one or more JDL's corresponding to findex. If findex is negative,
857// all run numbers are considered in one go (jdl). For non-negative indices
858// they correspond to the indices in the array fInputFiles.
859 if (!fInputFiles) return kFALSE;
860 TObjString *os;
a8739e8a 861 TString workdir = gGrid->GetHomeDirectory();
862 workdir += fGridWorkingDir;
d2a409b2 863
864 if (!fRunNumbers.Length() && !fRunRange[0]) {
865 // One jdl with no parameters in case input data is specified by name.
a8739e8a 866 TIter next(fInputFiles);
867 while ((os=(TObjString*)next()))
868 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 869 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 870 } else {
d2a409b2 871 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
872 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 873 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 874 }
875
876
877 // Generate the JDL as a string
878 TString sjdl = fGridJDL->Generate();
879 Int_t index;
880 index = sjdl.Index("Executable");
881 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
882 index = sjdl.Index("Split ");
883 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
884 index = sjdl.Index("SplitMaxInputFileNumber");
885 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
886 index = sjdl.Index("InputDataCollection");
887 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 888 index = sjdl.Index("InputFile");
889 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
890 index = sjdl.Index("InputDataList ");
891 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
892 index = sjdl.Index("InputDataListFormat");
893 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
894 index = sjdl.Index("Price");
895 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
896 index = sjdl.Index("Requirements");
897 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
898 index = sjdl.Index("Packages");
899 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
900 index = sjdl.Index("User =");
901 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
902 index = sjdl.Index("TTL");
903 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
904 index = sjdl.Index("OutputFile");
905 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
906 index = sjdl.Index("OutputDir");
907 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
908 index = sjdl.Index("OutputArchive");
909 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
910 index = sjdl.Index("MaxInitFailed");
911 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
912 index = sjdl.Index("MasterResubmitThreshold");
913 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
914 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
915 index = sjdl.Index("Validationcommand");
916 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
917 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
918 sjdl.ReplaceAll("(member", "\n (member");
919 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
920 sjdl.ReplaceAll("{", "{\n ");
921 sjdl.ReplaceAll("};", "\n};");
922 sjdl.ReplaceAll("{\n \n", "{\n");
923 sjdl.ReplaceAll("\n\n", "\n");
924 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
925 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 926 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 927 index = sjdl.Index("JDLVariables");
928 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
929 // Write jdl to file
a8739e8a 930 ofstream out;
d2a409b2 931 out.open(fJDLName.Data(), ios::out);
a8739e8a 932 if (out.bad()) {
d2a409b2 933 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 934 return kFALSE;
935 }
936 out << sjdl << endl;
937
938 // Copy jdl to grid workspace
939 if (!copy) {
940 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
941 } else {
d2a409b2 942 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
943 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 944 if (fProductionMode)
b5fe9cba 945 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 946 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 947 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 948 }
949 return kTRUE;
950}
951
c57f56b7 952//______________________________________________________________________________
5513444a 953Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 954{
955// Returns true if file exists.
5513444a 956 if (!gGrid) return kFALSE;
c57f56b7 957 TGridResult *res = gGrid->Ls(lfn);
958 if (!res) return kFALSE;
959 TMap *map = dynamic_cast<TMap*>(res->At(0));
960 if (!map) {
961 delete res;
962 return kFALSE;
963 }
964 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
965 if (!objs || !objs->GetString().Length()) {
966 delete res;
967 return kFALSE;
968 }
969 delete res;
970 return kTRUE;
971}
972
923e2ca5 973//______________________________________________________________________________
974Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
975{
976// Returns true if directory exists. Can be also a path.
977 if (!gGrid) return kFALSE;
978 // Check if dirname is a path
979 TString dirstripped = dirname;
980 dirstripped = dirstripped.Strip();
981 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
982 TString dir = gSystem->BaseName(dirstripped);
983 dir += "/";
984 TString path = gSystem->DirName(dirstripped);
985 TGridResult *res = gGrid->Ls(path, "-F");
986 if (!res) return kFALSE;
987 TIter next(res);
988 TMap *map;
989 TObject *obj;
990 while ((map=dynamic_cast<TMap*>(next()))) {
991 obj = map->GetValue("name");
992 if (!obj) break;
993 if (dir == obj->GetName()) {
994 delete res;
995 return kTRUE;
996 }
997 }
998 delete res;
999 return kFALSE;
1000}
1001
c57f56b7 1002//______________________________________________________________________________
1003void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1004{
1005// Check input data type.
1006 is_collection = kFALSE;
1007 is_xml = kFALSE;
1008 use_tags = kFALSE;
1009 if (!gGrid) {
1010 Error("CheckDataType", "No connection to grid");
1011 return;
1012 }
1013 is_collection = IsCollection(lfn);
1014 TString msg = "\n##### file: ";
1015 msg += lfn;
1016 if (is_collection) {
1017 msg += " type: raw_collection;";
1018 // special treatment for collections
1019 is_xml = kFALSE;
1020 // check for tag files in the collection
1021 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1022 if (!res) {
1023 msg += " using_tags: No (unknown)";
1024 Info("CheckDataType", msg.Data());
1025 return;
1026 }
1027 const char* typeStr = res->GetKey(0, "origLFN");
1028 if (!typeStr || !strlen(typeStr)) {
1029 msg += " using_tags: No (unknown)";
1030 Info("CheckDataType", msg.Data());
1031 return;
1032 }
1033 TString file = typeStr;
1034 use_tags = file.Contains(".tag");
1035 if (use_tags) msg += " using_tags: Yes";
1036 else msg += " using_tags: No";
1037 Info("CheckDataType", msg.Data());
1038 return;
1039 }
1040 TString slfn(lfn);
1041 slfn.ToLower();
1042 is_xml = slfn.Contains(".xml");
1043 if (is_xml) {
1044 // Open xml collection and check if there are tag files inside
1045 msg += " type: xml_collection;";
1046 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1047 if (!coll) {
1048 msg += " using_tags: No (unknown)";
1049 Info("CheckDataType", msg.Data());
1050 return;
1051 }
1052 TMap *map = coll->Next();
1053 if (!map) {
1054 msg += " using_tags: No (unknown)";
1055 Info("CheckDataType", msg.Data());
1056 return;
1057 }
1058 map = (TMap*)map->GetValue("");
1059 TString file;
1060 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1061 use_tags = file.Contains(".tag");
1062 delete coll;
1063 if (use_tags) msg += " using_tags: Yes";
1064 else msg += " using_tags: No";
1065 Info("CheckDataType", msg.Data());
1066 return;
1067 }
1068 use_tags = slfn.Contains(".tag");
1069 if (slfn.Contains(".root")) msg += " type: root file;";
1070 else msg += " type: unhnown file;";
1071 if (use_tags) msg += " using_tags: Yes";
1072 else msg += " using_tags: No";
1073 Info("CheckDataType", msg.Data());
1074}
1075
4e5c5506 1076//______________________________________________________________________________
1077void AliAnalysisAlien::EnablePackage(const char *package)
1078{
1079// Enables a par file supposed to exist in the current directory.
1080 TString pkg(package);
1081 pkg.ReplaceAll(".par", "");
1082 pkg += ".par";
1083 if (gSystem->AccessPathName(pkg)) {
1084 Error("EnablePackage", "Package %s not found", pkg.Data());
1085 return;
1086 }
1087 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1088 Info("EnablePackage", "AliEn plugin will use .par packages");
1089 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1090 if (!fPackages) {
1091 fPackages = new TObjArray();
1092 fPackages->SetOwner();
1093 }
1094 fPackages->Add(new TObjString(pkg));
1095}
1096
d2a409b2 1097//______________________________________________________________________________
1098const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1099{
1100// Get job status for all jobs with jobid>jobidstart.
1101 static char mstatus[20];
1102 mstatus[0] = '\0';
1103 nrunning = 0;
1104 nwaiting = 0;
1105 nerror = 0;
1106 ndone = 0;
1107 TGridJobStatusList *list = gGrid->Ps("");
1108 if (!list) return mstatus;
1109 Int_t nentries = list->GetSize();
1110 TGridJobStatus *status;
1111 Int_t pid;
1112 for (Int_t ijob=0; ijob<nentries; ijob++) {
1113 status = (TGridJobStatus *)list->At(ijob);
1114 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1115 if (pid<jobidstart) continue;
1116 if (pid == lastid) {
1117 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1118 }
1119 switch (status->GetStatus()) {
1120 case TGridJobStatus::kWAITING:
1121 nwaiting++; break;
1122 case TGridJobStatus::kRUNNING:
1123 nrunning++; break;
1124 case TGridJobStatus::kABORTED:
1125 case TGridJobStatus::kFAIL:
1126 case TGridJobStatus::kUNKNOWN:
1127 nerror++; break;
1128 case TGridJobStatus::kDONE:
1129 ndone++;
1130 }
1131 }
1132 list->Delete();
1133 delete list;
1134 return mstatus;
1135}
1136
c57f56b7 1137//______________________________________________________________________________
1138Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1139{
1140// Returns true if file is a collection. Functionality duplicated from
1141// TAlien::Type() because we don't want to directly depend on TAlien.
1142 if (!gGrid) {
1143 Error("IsCollection", "No connection to grid");
1144 return kFALSE;
1145 }
1146 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1147 if (!res) return kFALSE;
1148 const char* typeStr = res->GetKey(0, "type");
1149 if (!typeStr || !strlen(typeStr)) return kFALSE;
1150 if (!strcmp(typeStr, "collection")) return kTRUE;
1151 delete res;
1152 return kFALSE;
1153}
1154
16a4353c 1155//______________________________________________________________________________
1156void AliAnalysisAlien::Print(Option_t *) const
1157{
1158// Print current plugin settings.
923e2ca5 1159 Printf("### AliEn analysis plugin current settings ###");
1160 Printf("= Production mode:______________________________ %d", fProductionMode);
1161 Printf("= Version of API requested: ____________________ %s", fAPIVersion.Data());
1162 Printf("= Version of ROOT requested: ___________________ %s", fROOTVersion.Data());
1163 Printf("= Version of AliRoot requested: ________________ %s", fAliROOTVersion.Data());
16a4353c 1164 if (fUser.Length())
923e2ca5 1165 Printf("= User running the plugin: _____________________ %s", fUser.Data());
1166 Printf("= Grid workdir relative to user $HOME: _________ %s", fGridWorkingDir.Data());
1167 Printf("= Grid output directory relative to workdir: ___ %s", fGridOutputDir.Data());
1168 Printf("= Data base directory path requested: __________ %s", fGridDataDir.Data());
1169 Printf("= Data search pattern: _________________________ %s", fDataPattern.Data());
1170 Printf("= Input data format: ___________________________ %s", fInputFormat.Data());
16a4353c 1171 if (fRunNumbers.Length())
923e2ca5 1172 Printf("= Run numbers to be processed: _________________ %s", fRunNumbers.Data());
16a4353c 1173 if (fRunRange[0])
923e2ca5 1174 Printf("= Run range to be processed: ___________________ %d-%d", fRunRange[0], fRunRange[1]);
16a4353c 1175 if (!fRunRange[0] && !fRunNumbers.Length()) {
1176 TIter next(fInputFiles);
1177 TObject *obj;
1178 TString list;
1179 while ((obj=next())) list += obj->GetName();
923e2ca5 1180 Printf("= Input files to be processed: _________________ %s", list.Data());
16a4353c 1181 }
1182 if (TestBit(AliAnalysisGrid::kTest))
923e2ca5 1183 Printf("= Number of input files used in test mode: _____ %d", fNtestFiles);
1184 Printf("= List of output files to be registered: _______ %s", fOutputFiles.Data());
1185 Printf("= List of outputs going to be archived: ________ %s", fOutputArchive.Data());
1186 Printf("= List of outputs that should not be merged: ___ %s", fMergeExcludes.Data());
1187 Printf("=====================================================================");
1188 Printf("= Job price: ___________________________________ %d", fPrice);
1189 Printf("= Time to live (TTL): __________________________ %d", fTTL);
1190 Printf("= Max files per subjob: ________________________ %d", fSplitMaxInputFileNumber);
16a4353c 1191 if (fMaxInitFailed>0)
923e2ca5 1192 Printf("= Max number of subjob fails to kill: __________ %d", fMaxInitFailed);
16a4353c 1193 if (fMasterResubmitThreshold>0)
923e2ca5 1194 Printf("= Resubmit master job if failed subjobs >_______ %d", fMasterResubmitThreshold);
319593fb 1195 if (fNrunsPerMaster>0)
923e2ca5 1196 Printf("= Number of runs per master job: _______________ %d", fNrunsPerMaster);
1197 Printf("= Number of files in one chunk to be merged: ___ %d", fMaxMergeFiles);
1198 Printf("= Name of the generated execution script: ______ %s",fExecutable.Data());
16a4353c 1199 if (fArguments.Length())
923e2ca5 1200 Printf("= Arguments for the execution script: __________ %s",fArguments.Data());
1201 Printf("= Name of the generated analysis macro: ________ %s",fAnalysisMacro.Data());
1202 Printf("= User analysis files to be deployed: __________ %s",fAnalysisSource.Data());
1203 Printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>",fAdditionalLibs.Data());
1204 Printf("= Master jobs split mode: ______________________ %s",fSplitMode.Data());
16a4353c 1205 if (fDatasetName)
923e2ca5 1206 Printf("= Custom name for the dataset to be created: ___ %s", fDatasetName.Data());
1207 Printf("= Name of the generated JDL: ___________________ %s", fJDLName.Data());
16a4353c 1208 if (fIncludePath.Data())
923e2ca5 1209 Printf("= Include path for runtime task compilation: ___ %s", fIncludePath.Data());
16a4353c 1210 if (fCloseSE.Length())
923e2ca5 1211 Printf("= Force job outputs to storage element: ________ %s", fCloseSE.Data());
16a4353c 1212 if (fFriendChainName.Length())
923e2ca5 1213 Printf("= Open friend chain file on worker: ____________ %s", fFriendChainName.Data());
16a4353c 1214 if (fPackages) {
1215 TIter next(fPackages);
1216 TObject *obj;
1217 TString list;
1218 while ((obj=next())) list += obj->GetName();
923e2ca5 1219 Printf("= Par files to be used: ________________________ %s", list.Data());
16a4353c 1220 }
1221}
1222
c57f56b7 1223//______________________________________________________________________________
1224void AliAnalysisAlien::SetDefaults()
1225{
1226// Set default values for everything. What cannot be filled will be left empty.
1227 if (fGridJDL) delete fGridJDL;
1228 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1229 fPrice = 1;
1230 fTTL = 30000;
1231 fSplitMaxInputFileNumber = 100;
1232 fMaxInitFailed = 0;
1233 fMasterResubmitThreshold = 0;
bb885a9e 1234 fNtestFiles = 10;
a8739e8a 1235 fRunRange[0] = 0;
1236 fRunRange[1] = 0;
319593fb 1237 fNrunsPerMaster = 1;
16a4353c 1238 fMaxMergeFiles = 100;
c57f56b7 1239 fRunNumbers = "";
1240 fExecutable = "analysis.sh";
1241 fArguments = "";
1242 fAnalysisMacro = "myAnalysis.C";
1243 fAnalysisSource = "";
1244 fAdditionalLibs = "";
1245 fSplitMode = "se";
1246 fAPIVersion = "";
1247 fROOTVersion = "";
1248 fAliROOTVersion = "";
1249 fUser = ""; // Your alien user name
1250 fGridWorkingDir = "";
1251 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1252 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1253 fFriendChainName = "";
c57f56b7 1254 fGridOutputDir = "output";
1255 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1256 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1257 fInputFormat = "xml-single";
1258 fJDLName = "analysis.jdl";
c6cb3634 1259 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1260 fMergeExcludes = "";
c57f56b7 1261}
1262
1263//______________________________________________________________________________
1264Bool_t AliAnalysisAlien::MergeOutputs()
1265{
1266// Merge analysis outputs existing in the AliEn space.
1267 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1268 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1269 if (!Connect()) {
1270 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1271 return kFALSE;
1272 }
1273 // Get the output path
d2a409b2 1274 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1275 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1276 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1277 return kFALSE;
1278 }
1279 if (!fOutputFiles.Length()) {
1280 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1281 return kFALSE;
1282 }
1283 TObjArray *list = fOutputFiles.Tokenize(" ");
1284 TIter next(list);
1285 TObjString *str;
1286 TString command;
1287 TString output_file;
16a4353c 1288 TString output_chunk;
1289 TString previous_chunk;
1290 Int_t count_chunk = 0;
1291 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1292 Bool_t merged = kTRUE;
1293 while((str=(TObjString*)next())) {
1294 output_file = str->GetString();
1295 Int_t index = output_file.Index("@");
1296 if (index > 0) output_file.Remove(index);
319593fb 1297 // Skip already merged outputs
1298 if (!gSystem->AccessPathName(output_file)) {
1299 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1300 continue;
1301 }
bb885a9e 1302 if (fMergeExcludes.Length() &&
1303 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1304 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1305 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
923e2ca5 1306 Printf("command: %s", command.Data());
c57f56b7 1307 TGridResult *res = gGrid->Command(command);
1308 if (!res) continue;
1309 TFileMerger *fm = 0;
1310 TIter nextmap(res);
ab254fd1 1311 TMap *map = 0;
16a4353c 1312 previous_chunk = "";
1313 count_chunk = 0;
319593fb 1314 // Check if there is a merge operation to resume
1315 output_chunk = output_file;
1316 output_chunk.ReplaceAll(".root", "_*.root");
1317 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1318 while (1) {
1319 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1320 if (!map) {
1321 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1322 delete res;
1323 return kFALSE;
1324 }
1325 output_chunk = output_file;
1326 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
923e2ca5 1327 Printf("%s", output_chunk.Data());
319593fb 1328 count_chunk++;
1329 if (gSystem->AccessPathName(output_chunk)) continue;
1330 // Merged file with chunks up to <count_chunk> found
923e2ca5 1331 Printf("Resume merging of <%s> from <%s>", output_file.Data(), output_chunk.Data());
319593fb 1332 previous_chunk = output_chunk;
1333 break;
1334 }
1335 }
16a4353c 1336 count_zero = fMaxMergeFiles;
c57f56b7 1337 while ((map=(TMap*)nextmap())) {
16a4353c 1338 // Loop 'find' results and get next LFN
1339 if (count_zero == fMaxMergeFiles) {
1340 // First file in chunk - create file merger and add previous chunk if any.
1341 fm = new TFileMerger(kFALSE);
1342 fm->SetFastMethod(kTRUE);
1343 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1344 output_chunk = output_file;
1345 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1346 }
1347 // If last file found, put merged results in the output file
1348 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1349 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1350 if (!objs || !objs->GetString().Length()) {
16a4353c 1351 // Nothing found - skip this output
c57f56b7 1352 delete res;
16a4353c 1353 delete fm;
1354 break;
1355 }
1356 // Add file to be merged and decrement chunk counter.
1357 fm->AddFile(objs->GetString());
1358 count_zero--;
1359 if (count_zero==0 || map == res->Last()) {
1360 fm->OutputFile(output_chunk);
1361 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1362 // Nothing found - skip this output
1363 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1364 delete res;
1365 delete fm;
1366 break;
1367 }
1368 // Merge the outputs, then go to next chunk
1369 if (!fm->Merge()) {
1370 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1371 delete res;
1372 delete fm;
1373 merged = kFALSE;
1374 break;
1375 } else {
1376 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1377 gSystem->Unlink(previous_chunk);
16a4353c 1378 }
1379 if (map == res->Last()) {
1380 delete res;
1381 delete fm;
1382 break;
1383 }
1384 count_chunk++;
1385 count_zero = fMaxMergeFiles;
1386 previous_chunk = output_chunk;
c57f56b7 1387 }
c57f56b7 1388 }
c57f56b7 1389 }
1390 if (!merged) {
1391 Error("MergeOutputs", "Terminate() will NOT be executed");
1392 }
1393 return merged;
1394}
1395
bb885a9e 1396//______________________________________________________________________________
1397void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1398{
1399// Use the output files connected to output containers from the analysis manager
1400// rather than the files defined by SetOutputFiles
1401 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1402 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1403 analysis manager");
1404 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1405}
1406
5513444a 1407//______________________________________________________________________________
1408Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1409{
1410// Start remote grid analysis.
1411
43da816a 1412 // Check if output files have to be taken from the analysis manager
1413 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1414 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1415 if (!mgr || !mgr->IsInitialized()) {
1416 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1417 return kFALSE;
43da816a 1418 }
1419 fOutputFiles = "";
1420 TIter next(mgr->GetOutputs());
1421 AliAnalysisDataContainer *output;
1422 while ((output=(AliAnalysisDataContainer*)next())) {
1423 const char *filename = output->GetFileName();
1424 if (!(strcmp(filename, "default"))) {
1425 if (!mgr->GetOutputEventHandler()) continue;
1426 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1427 }
1428 if (fOutputFiles.Length()) fOutputFiles += " ";
1429 fOutputFiles += filename;
1430 }
c07b9ce2 1431 // Add extra files registered to the analysis manager
1432 if (mgr->GetExtraFiles().Length()) {
1433 if (fOutputFiles.Length()) fOutputFiles += " ";
1434 fOutputFiles += mgr->GetExtraFiles();
1435 }
43da816a 1436 }
f7b1cbc2 1437// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1438 if (TestBit(AliAnalysisGrid::kOffline)) {
1439 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1440 \n there nor any job run. You can revise the JDL and analysis \
1441 \n macro then run the same in \"submit\" mode.");
1442 } else if (TestBit(AliAnalysisGrid::kTest)) {
1443 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1444 \n dataset.");
1445 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1446 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1447 \n space and job submitted.");
1448 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1449 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1450 return kTRUE;
c57f56b7 1451 } else {
1452 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1453 }
1454
1455 if (!Connect()) {
1456 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1457 return kFALSE;
16a4353c 1458 }
1459 Print();
c57f56b7 1460 if (!CheckInputData()) {
1461 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1462 return kFALSE;
c57f56b7 1463 }
1464 CreateDataset(fDataPattern);
1465 WriteAnalysisFile();
1466 WriteAnalysisMacro();
1467 WriteExecutable();
1468 WriteValidationScript();
5513444a 1469 if (!CreateJDL()) return kFALSE;
1470 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1471 if (TestBit(AliAnalysisGrid::kTest)) {
1472 // Locally testing the analysis
1473 Info("StartAnalysis", "\n_______________________________________________________________________ \
1474 \n Running analysis script in a daughter shell as on a worker node \
1475 \n_______________________________________________________________________");
1476 TObjArray *list = fOutputFiles.Tokenize(" ");
1477 TIter next(list);
1478 TObjString *str;
1479 TString output_file;
1480 while((str=(TObjString*)next())) {
1481 output_file = str->GetString();
1482 Int_t index = output_file.Index("@");
1483 if (index > 0) output_file.Remove(index);
43da816a 1484 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1485 }
1486 delete list;
1487 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1488 TString validationScript = fExecutable;
1489 validationScript.ReplaceAll(".sh", "_validation.sh");
1490 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1491// gSystem->Exec("cat stdout");
5513444a 1492 return kFALSE;
c57f56b7 1493 }
5513444a 1494 // Check if submitting is managed by LPM manager
a3e84053 1495 if (fProductionMode) {
5513444a 1496 TString prodfile = fJDLName;
1497 prodfile.ReplaceAll(".jdl", ".prod");
1498 WriteProductionFile(prodfile);
1499 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1500 return kFALSE;
1501 }
a8739e8a 1502 // Submit AliEn job(s)
d2a409b2 1503 gGrid->Cd(fGridOutputDir);
a8739e8a 1504 TGridResult *res;
c57f56b7 1505 TString jobID = "";
d2a409b2 1506 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1507 // Submit a given xml or a set of runs
a8739e8a 1508 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
923e2ca5 1509 Printf("*************************** %s",Form("submit %s", fJDLName.Data()));
a8739e8a 1510 if (res) {
1511 const char *cjobId = res->GetKey(0,"jobId");
1512 if (!cjobId) {
1513 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1514 return kFALSE;
a8739e8a 1515 } else {
1516 Info("StartAnalysis", "\n_______________________________________________________________________ \
1517 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1518 \n_______________________________________________________________________",
1519 fJDLName.Data(), cjobId);
1520 jobID = cjobId;
1521 }
1522 delete res;
1523 }
1524 } else {
d2a409b2 1525 // Submit for a range of enumeration of runs.
1526 Submit();
c57f56b7 1527 }
a8739e8a 1528
c57f56b7 1529 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1530 \n You may exit at any time and terminate the job later using the option <terminate> \
1531 \n ##################################################################################", jobID.Data());
bb885a9e 1532 gSystem->Exec("aliensh");
5513444a 1533 return kTRUE;
c57f56b7 1534}
1535
d2a409b2 1536//______________________________________________________________________________
1537void AliAnalysisAlien::Submit()
1538{
1539// Submit all master jobs.
1540 Int_t nmasterjobs = fInputFiles->GetEntries();
1541 Long_t tshoot = gSystem->Now();
1542 if (!fNsubmitted) SubmitNext();
1543 while (fNsubmitted < nmasterjobs) {
1544 Long_t now = gSystem->Now();
1545 if ((now-tshoot)>30000) {
1546 tshoot = now;
1547 SubmitNext();
1548 }
1549 }
1550}
1551
1552//______________________________________________________________________________
1553void AliAnalysisAlien::SubmitNext()
1554{
1555// Submit next bunch of master jobs if the queue is free.
1556 static Bool_t iscalled = kFALSE;
1557 static Int_t firstmaster = 0;
1558 static Int_t lastmaster = 0;
1559 static Int_t npermaster = 0;
1560 if (iscalled) return;
1561 iscalled = kTRUE;
1562 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1563 Int_t ntosubmit = 0;
1564 TGridResult *res;
1565 TString jobID = "";
1566 if (!fNsubmitted) ntosubmit = 1;
1567 else {
1568 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
923e2ca5 1569 Printf("=== master %d: %s", lastmaster, status.Data());
d2a409b2 1570 // If last master not split, just return
1571 if (status != "SPLIT") {iscalled = kFALSE; return;}
1572 // No more than 100 waiting jobs
1573 if (nwaiting>100) {iscalled = kFALSE; return;}
1574 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1575 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
923e2ca5 1576 Printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs",
d2a409b2 1577 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1578 }
1579 Int_t nmasterjobs = fInputFiles->GetEntries();
1580 for (Int_t i=0; i<ntosubmit; i++) {
1581 // Submit for a range of enumeration of runs.
1582 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1583 TString query;
1584 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
923e2ca5 1585 Printf("********* %s",query.Data());
d2a409b2 1586 res = gGrid->Command(query);
1587 if (res) {
98ca124f 1588 TString cjobId1 = res->GetKey(0,"jobId");
1589 if (!cjobId1.Length()) {
d2a409b2 1590 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1591 iscalled = kFALSE;
1592 return;
1593 } else {
1594 Info("StartAnalysis", "\n_______________________________________________________________________ \
1595 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1596 \n_______________________________________________________________________",
98ca124f 1597 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1598 jobID += cjobId1;
1599 jobID += " ";
98ca124f 1600 lastmaster = cjobId1.Atoi();
d2a409b2 1601 if (!firstmaster) firstmaster = lastmaster;
1602 fNsubmitted++;
1603 }
1604 delete res;
1605 }
1606 }
1607 iscalled = kFALSE;
1608}
1609
c57f56b7 1610//______________________________________________________________________________
1611void AliAnalysisAlien::WriteAnalysisFile()
1612{
f10e8481 1613// Write current analysis manager into the file <analysisFile>
1614 TString analysisFile = fExecutable;
1615 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1616 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1617 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1618 if (!mgr || !mgr->IsInitialized()) {
1619 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1620 return;
1621 }
1622 // Check analysis type
1623 TObject *handler;
1624 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1625 handler = (TObject*)mgr->GetInputEventHandler();
1626 if (handler) {
1627 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1628 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1629 }
1630 TDirectory *cdir = gDirectory;
f10e8481 1631 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1632 if (file) {
1633 mgr->Write();
1634 delete file;
1635 }
1636 if (cdir) cdir->cd();
f10e8481 1637 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1638 }
1639 Bool_t copy = kTRUE;
1640 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1641 if (copy) {
1642 CdWork();
1643 TString workdir = gGrid->GetHomeDirectory();
1644 workdir += fGridWorkingDir;
f10e8481 1645 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1646 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1647 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1648 }
1649}
1650
1651//______________________________________________________________________________
1652void AliAnalysisAlien::WriteAnalysisMacro()
1653{
1654// Write the analysis macro that will steer the analysis in grid mode.
1655 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1656 ofstream out;
1657 out.open(fAnalysisMacro.Data(), ios::out);
1658 if (!out.good()) {
1659 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1660 return;
1661 }
1662 TString func = fAnalysisMacro;
1663 TString type = "ESD";
1664 TString comment = "// Analysis using ";
1665 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1666 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1667 type = "AOD";
1668 comment += "AOD";
1669 }
0df6ccf2 1670 if (type!="AOD" && fFriendChainName!="") {
1671 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1672 return;
1673 }
c57f56b7 1674 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1675 else comment += " data";
1676 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1677 func.ReplaceAll(".C", "");
1678 out << "void " << func.Data() << "()" << endl;
1679 out << "{" << endl;
1680 out << comment.Data() << endl;
1681 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1682 out << " TStopwatch timer;" << endl;
1683 out << " timer.Start();" << endl << endl;
c57f56b7 1684 out << "// load base root libraries" << endl;
1685 out << " gSystem->Load(\"libTree\");" << endl;
1686 out << " gSystem->Load(\"libGeom\");" << endl;
1687 out << " gSystem->Load(\"libVMC\");" << endl;
1688 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1689 out << "// Load analysis framework libraries" << endl;
4e5c5506 1690 if (!fPackages) {
4e5c5506 1691 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1692 out << " gSystem->Load(\"libESD\");" << endl;
1693 out << " gSystem->Load(\"libAOD\");" << endl;
1694 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1695 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1696 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1697 } else {
4e5c5506 1698 TIter next(fPackages);
1699 TObject *obj;
57377eb5 1700 TString pkgname;
1701 Bool_t hasSTEERBase = kFALSE;
1702 Bool_t hasESD = kFALSE;
1703 Bool_t hasAOD = kFALSE;
1704 Bool_t hasANALYSIS = kFALSE;
1705 Bool_t hasANALYSISalice = kFALSE;
1706 Bool_t hasCORRFW = kFALSE;
1707 while ((obj=next())) {
1708 pkgname = obj->GetName();
4478e6f1 1709 if (pkgname == "STEERBase" ||
1710 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1711 if (pkgname == "ESD" ||
1712 pkgname == "ESD.par") hasESD = kTRUE;
1713 if (pkgname == "AOD" ||
1714 pkgname == "AOD.par") hasAOD = kTRUE;
1715 if (pkgname == "ANALYSIS" ||
1716 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1717 if (pkgname == "ANALYSISalice" ||
1718 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1719 if (pkgname == "CORRFW" ||
1720 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1721 }
1722 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1723 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1724 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1725 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1726 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1727 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1728 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1729 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1730 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1731 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1732 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1733 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1734 out << "// Compile other par packages" << endl;
1735 next.Reset();
fcc9bb6f 1736 while ((obj=next())) {
1737 pkgname = obj->GetName();
4478e6f1 1738 if (pkgname == "STEERBase" ||
1739 pkgname == "STEERBase.par" ||
1740 pkgname == "ESD" ||
1741 pkgname == "ESD.par" ||
1742 pkgname == "AOD" ||
1743 pkgname == "AOD.par" ||
1744 pkgname == "ANALYSIS" ||
1745 pkgname == "ANALYSIS.par" ||
1746 pkgname == "ANALYSISalice" ||
1747 pkgname == "ANALYSISalice.par" ||
1748 pkgname == "CORRFW" ||
1749 pkgname == "CORRFW.par") continue;
4e5c5506 1750 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1751 }
4e5c5506 1752 }
43da816a 1753 out << "// include path" << endl;
1754 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1755 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1756 if (fAdditionalLibs.Length()) {
1757 out << "// Add aditional AliRoot libraries" << endl;
1758 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1759 TIter next(list);
1760 TObjString *str;
1761 while((str=(TObjString*)next())) {
1762 if (str->GetString().Contains(".so"))
1763 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1764 }
1765 if (list) delete list;
1766 }
1767 out << endl;
c57f56b7 1768 out << "// analysis source to be compiled at runtime (if any)" << endl;
1769 if (fAnalysisSource.Length()) {
1770 TObjArray *list = fAnalysisSource.Tokenize(" ");
1771 TIter next(list);
1772 TObjString *str;
1773 while((str=(TObjString*)next())) {
1774 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1775 }
1776 if (list) delete list;
1777 }
1778 out << endl;
1779 out << "// connect to AliEn and make the chain" << endl;
1780 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1781 if (IsUsingTags()) {
1782 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1783 } else {
76535c98 1784 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1785 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1786 } else {
1787 out << " // Check if the macro to create the chain was provided" << endl;
1788 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1789 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1790 out << " return;" << endl;
1791 out << " }" << endl;
1792 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1793 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1794 }
c57f56b7 1795 }
1796 out << "// read the analysis manager from file" << endl;
f10e8481 1797 TString analysisFile = fExecutable;
1798 analysisFile.ReplaceAll(".sh", ".root");
1799 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1800 out << " if (!file) return;" << endl;
1801 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1802 out << " AliAnalysisManager *mgr = 0;" << endl;
1803 out << " TKey *key;" << endl;
1804 out << " while ((key=(TKey*)nextkey())) {" << endl;
1805 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1806 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1807 out << " };" << endl;
1808 out << " if (!mgr) {" << endl;
f10e8481 1809 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1810 out << " return;" << endl;
1811 out << " }" << endl << endl;
1812 out << " mgr->PrintStatus();" << endl;
1813 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1814 out << " timer.Stop();" << endl;
1815 out << " timer.Print();" << endl;
c57f56b7 1816 out << "}" << endl << endl;
1817 if (IsUsingTags()) {
1818 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1819 out << "{" << endl;
1820 out << "// Create a chain using tags from the xml file." << endl;
1821 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1822 out << " if (!coll) {" << endl;
1823 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1824 out << " return NULL;" << endl;
1825 out << " }" << endl;
1826 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1827 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1828 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1829 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1830 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1831 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1832 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1833 out << " // Check if the cuts configuration file was provided" << endl;
1834 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1835 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1836 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1837 out << " }" << endl;
0df6ccf2 1838 if (fFriendChainName=="") {
1839 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1840 } else {
1841 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1842 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1843 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1844 }
c57f56b7 1845 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1846 out << " chain->ls();" << endl;
1847 out << " return chain;" << endl;
fcc9bb6f 1848 out << "}" << endl << endl;
c57f56b7 1849 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1850 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1851 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1852 msg += " AliLHCTagCuts *lhcCuts,\n";
1853 msg += " AliDetectorTagCuts *detCuts,\n";
1854 msg += " AliEventTagCuts *evCuts)";
1855 Info("WriteAnalysisMacro", msg.Data());
1856 }
0df6ccf2 1857 }
1858 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1859 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1860 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1861 out << "{" << endl;
1862 out << "// Create a chain using url's from xml file" << endl;
1863 out << " TString treename = type;" << endl;
1864 out << " treename.ToLower();" << endl;
1865 out << " treename += \"Tree\";" << endl;
e02fee64 1866 out << " printf(\"***************************************\\n\");" << endl;
1867 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1868 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1869 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1870 out << " if (!coll) {" << endl;
1871 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1872 out << " return NULL;" << endl;
1873 out << " }" << endl;
1874 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1875 if(fFriendChainName!="") {
1876 out << " TChain *chainFriend = new TChain(treename);" << endl;
1877 }
c57f56b7 1878 out << " coll->Reset();" << endl;
0df6ccf2 1879 out << " while (coll->Next()) {" << endl;
1880 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1881 if(fFriendChainName!="") {
1882 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1883 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1884 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1885 out << " chainFriend->Add(fileFriend.Data());" << endl;
1886 }
1887 out << " }" << endl;
c57f56b7 1888 out << " if (!chain->GetNtrees()) {" << endl;
1889 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1890 out << " return NULL;" << endl;
1891 out << " }" << endl;
0df6ccf2 1892 if(fFriendChainName!="") {
1893 out << " chain->AddFriend(chainFriend);" << endl;
1894 }
c57f56b7 1895 out << " return chain;" << endl;
fcc9bb6f 1896 out << "}" << endl << endl;
c57f56b7 1897 }
4e5c5506 1898 if (fPackages) {
fcc9bb6f 1899 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1900 out << "Bool_t SetupPar(const char *package) {" << endl;
1901 out << "// Compile the package and set it up." << endl;
1902 out << " TString pkgdir = package;" << endl;
1903 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1904 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1905 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1906 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1907 out << " // Check for BUILD.sh and execute" << endl;
1908 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1909 out << " printf(\"*******************************\\n\");" << endl;
1910 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1911 out << " printf(\"*******************************\\n\");" << endl;
1912 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1913 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1914 out << " gSystem->ChangeDirectory(cdir);" << endl;
1915 out << " return kFALSE;" << endl;
1916 out << " }" << endl;
1917 out << " } else {" << endl;
fcc9bb6f 1918 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1919 out << " gSystem->ChangeDirectory(cdir);" << endl;
1920 out << " return kFALSE;" << endl;
1921 out << " }" << endl;
1922 out << " // Check for SETUP.C and execute" << endl;
1923 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1924 out << " printf(\"*******************************\\n\");" << endl;
1925 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1926 out << " printf(\"*******************************\\n\");" << endl;
1927 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1928 out << " } else {" << endl;
fcc9bb6f 1929 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1930 out << " gSystem->ChangeDirectory(cdir);" << endl;
1931 out << " return kFALSE;" << endl;
1932 out << " }" << endl;
1933 out << " // Restore original workdir" << endl;
1934 out << " gSystem->ChangeDirectory(cdir);" << endl;
1935 out << " return kTRUE;" << endl;
1936 out << "}" << endl;
1937 }
c57f56b7 1938 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1939 }
1940 Bool_t copy = kTRUE;
1941 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1942 if (copy) {
1943 CdWork();
1944 TString workdir = gGrid->GetHomeDirectory();
1945 workdir += fGridWorkingDir;
1946 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1947 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1948 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1949 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1950 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1951 }
1952 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1953 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1954 }
1955}
1956
1957//______________________________________________________________________________
1958void AliAnalysisAlien::WriteExecutable()
1959{
1960// Generate the alien executable script.
1961 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1962 ofstream out;
1963 out.open(fExecutable.Data(), ios::out);
1964 if (out.bad()) {
5513444a 1965 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1966 return;
1967 }
1968 out << "#!/bin/bash" << endl;
c57f56b7 1969 out << "echo \"=========================================\"" << endl;
1970 out << "echo \"############## PATH : ##############\"" << endl;
1971 out << "echo $PATH" << endl;
1972 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1973 out << "echo $LD_LIBRARY_PATH" << endl;
1974 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1975 out << "echo $ROOTSYS" << endl;
1976 out << "echo \"############## which root : ##############\"" << endl;
1977 out << "which root" << endl;
1978 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1979 out << "echo $ALICE_ROOT" << endl;
1980 out << "echo \"############## which aliroot : ##############\"" << endl;
1981 out << "which aliroot" << endl;
1982 out << "echo \"=========================================\"" << endl << endl;
1983// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1984 out << "root -b -q ";
1985 out << fAnalysisMacro.Data() << endl << endl;
1986 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1987 }
1988 Bool_t copy = kTRUE;
1989 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1990 if (copy) {
1991 CdWork();
1992 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 1993 TString bindir = Form("%s/bin", workdir.Data());
1994 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 1995 workdir += fGridWorkingDir;
1996 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1997 if (FileExists(executable)) gGrid->Rm(executable);
1998 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1999 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2000 }
2001}
2002
2003//______________________________________________________________________________
5513444a 2004void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2005{
2006// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2007// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2008// Next lines: full_path_to_dataset XXX (XXX is a string)
2009// To submit, one has to: submit jdl XXX for all lines
2010 ofstream out;
2011 out.open(filename, ios::out);
2012 if (out.bad()) {
2013 Error("WriteProductionFile", "Bad file name: %s", filename);
2014 return;
2015 }
2016 TString workdir = gGrid->GetHomeDirectory();
2017 workdir += fGridWorkingDir;
f5e8c702 2018 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2019 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2020 out << locjdl << " " << njobspermaster << endl;
5513444a 2021 Int_t nmasterjobs = fInputFiles->GetEntries();
2022 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2023 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2024 }
2025 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2026 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2027}
2028
2029//______________________________________________________________________________
c57f56b7 2030void AliAnalysisAlien::WriteValidationScript()
2031{
2032// Generate the alien validation script.
2033 // Generate the validation script
2034 TObjString *os;
0d5d317c 2035 TString validationScript = fExecutable;
2036 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2037 if (!Connect()) {
2038 Error("WriteValidationScript", "Alien connection required");
2039 return;
2040 }
2041 TString out_stream = "";
2042 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2043 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2044 ofstream out;
0d5d317c 2045 out.open(validationScript, ios::out);
c57f56b7 2046 out << "#!/bin/bash" << endl;
2047 out << "##################################################" << endl;
2048 out << "validateout=`dirname $0`" << endl;
2049 out << "validatetime=`date`" << endl;
2050 out << "validated=\"0\";" << endl;
2051 out << "error=0" << endl;
2052 out << "if [ -z $validateout ]" << endl;
2053 out << "then" << endl;
2054 out << " validateout=\".\"" << endl;
2055 out << "fi" << endl << endl;
2056 out << "cd $validateout;" << endl;
2057 out << "validateworkdir=`pwd`;" << endl << endl;
2058 out << "echo \"*******************************************************\"" << out_stream << endl;
2059 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2060 out << "" << endl;
2061 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2062 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2063 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2064 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2065 out << "ls -la ./" << out_stream << endl;
2066 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2067 out << "##################################################" << endl;
ebec370a 2068
2069 out << "" << endl;
2070 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2071 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2072 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2073 out << "" << endl;
2074
2075 out << "if [ ! -f stderr ] ; then" << endl;
2076 out << " error=1" << endl;
2077 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2078 out << " echo \"Error = $error\" " << out_stream << endl;
2079 out << "fi" << endl;
2080
2081 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2082 out << " error=1" << endl;
2083 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2084 out << " echo \"$parArch\" " << out_stream << endl;
2085 out << " echo \"Error = $error\" " << out_stream << endl;
2086 out << "fi" << endl;
2087
2088 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2089 out << " error=1" << endl;
2090 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2091 out << " echo \"$segViol\" " << out_stream << endl;
2092 out << " echo \"Error = $error\" " << out_stream << endl;
2093 out << "fi" << endl;
2094
2095 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2096 out << " error=1" << endl;
2097 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2098 out << " echo \"$segFault\" " << out_stream << endl;
2099 out << " echo \"Error = $error\" " << out_stream << endl;
2100 out << "fi" << endl;
2101
2102 // Part dedicated to the specific analyses running into the train
2103
c57f56b7 2104 TObjArray *arr = fOutputFiles.Tokenize(" ");
2105 TIter next1(arr);
2106 TString output_file;
2107 while ((os=(TObjString*)next1())) {
2108 output_file = os->GetString();
2109 Int_t index = output_file.Index("@");
2110 if (index > 0) output_file.Remove(index);
2111 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2112 out << " error=1" << endl;
2113 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2114 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2115 out << "fi" << endl;
2116 }
2117 delete arr;
923e2ca5 2118 out << "if ! [ -f outputs_valid ] ; then" << endl;
2119 out << " error=1" << endl;
2120 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2121 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2122 out << "fi" << endl;
2123
c57f56b7 2124 out << "if [ $error = 0 ] ; then" << endl;
2125 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2126 out << "fi" << endl;
2127
2128 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2129 out << "echo \"*******************************************************\"" << out_stream << endl;
2130 out << "cd -" << endl;
2131 out << "exit $error" << endl;
2132 }
2133 Bool_t copy = kTRUE;
2134 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2135 if (copy) {
2136 CdWork();
2137 TString workdir = gGrid->GetHomeDirectory();
2138 workdir += fGridWorkingDir;
0d5d317c 2139 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2140 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2141 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2142 }
2143}