]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
adding macro to clean datasets on CAF
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
0a1c1f7f 59 fExecutableCommand(),
c57f56b7 60 fArguments(),
61 fAnalysisMacro(),
62 fAnalysisSource(),
63 fAdditionalLibs(),
64 fSplitMode(),
65 fAPIVersion(),
66 fROOTVersion(),
67 fAliROOTVersion(),
68 fUser(),
69 fGridWorkingDir(),
70 fGridDataDir(),
71 fDataPattern(),
72 fGridOutputDir(),
73 fOutputArchive(),
74 fOutputFiles(),
75 fInputFormat(),
e7c71df0 76 fDatasetName(),
c57f56b7 77 fJDLName(),
bb885a9e 78 fMergeExcludes(),
f965131e 79 fIncludePath(),
bb885a9e 80 fCloseSE(),
0df6ccf2 81 fFriendChainName(),
c6cb3634 82 fJobTag(),
4e5c5506 83 fInputFiles(0),
84 fPackages(0)
c57f56b7 85{
86// Dummy ctor.
87 SetDefaults();
88}
89
90//______________________________________________________________________________
91AliAnalysisAlien::AliAnalysisAlien(const char *name)
92 :AliAnalysisGrid(name),
93 fGridJDL(NULL),
94 fPrice(0),
95 fTTL(0),
96 fSplitMaxInputFileNumber(0),
97 fMaxInitFailed(0),
98 fMasterResubmitThreshold(0),
bb885a9e 99 fNtestFiles(0),
319593fb 100 fNrunsPerMaster(0),
16a4353c 101 fMaxMergeFiles(0),
d2a409b2 102 fNsubmitted(0),
a3e84053 103 fProductionMode(0),
c57f56b7 104 fRunNumbers(),
105 fExecutable(),
0a1c1f7f 106 fExecutableCommand(),
c57f56b7 107 fArguments(),
108 fAnalysisMacro(),
109 fAnalysisSource(),
110 fAdditionalLibs(),
111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
115 fUser(),
116 fGridWorkingDir(),
117 fGridDataDir(),
118 fDataPattern(),
119 fGridOutputDir(),
120 fOutputArchive(),
121 fOutputFiles(),
122 fInputFormat(),
e7c71df0 123 fDatasetName(),
c57f56b7 124 fJDLName(),
bb885a9e 125 fMergeExcludes(),
f965131e 126 fIncludePath(),
bb885a9e 127 fCloseSE(),
0df6ccf2 128 fFriendChainName(),
c6cb3634 129 fJobTag(),
4e5c5506 130 fInputFiles(0),
131 fPackages(0)
c57f56b7 132{
133// Default ctor.
134 SetDefaults();
135}
136
137//______________________________________________________________________________
138AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
139 :AliAnalysisGrid(other),
140 fGridJDL(NULL),
141 fPrice(other.fPrice),
142 fTTL(other.fTTL),
143 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
144 fMaxInitFailed(other.fMaxInitFailed),
145 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 146 fNtestFiles(other.fNtestFiles),
319593fb 147 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 148 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 149 fNsubmitted(other.fNsubmitted),
a3e84053 150 fProductionMode(other.fProductionMode),
c57f56b7 151 fRunNumbers(other.fRunNumbers),
152 fExecutable(other.fExecutable),
0a1c1f7f 153 fExecutableCommand(other.fExecutableCommand),
c57f56b7 154 fArguments(other.fArguments),
155 fAnalysisMacro(other.fAnalysisMacro),
156 fAnalysisSource(other.fAnalysisSource),
157 fAdditionalLibs(other.fAdditionalLibs),
158 fSplitMode(other.fSplitMode),
159 fAPIVersion(other.fAPIVersion),
160 fROOTVersion(other.fROOTVersion),
161 fAliROOTVersion(other.fAliROOTVersion),
162 fUser(other.fUser),
163 fGridWorkingDir(other.fGridWorkingDir),
164 fGridDataDir(other.fGridDataDir),
165 fDataPattern(other.fDataPattern),
166 fGridOutputDir(other.fGridOutputDir),
167 fOutputArchive(other.fOutputArchive),
168 fOutputFiles(other.fOutputFiles),
169 fInputFormat(other.fInputFormat),
e7c71df0 170 fDatasetName(other.fDatasetName),
c57f56b7 171 fJDLName(other.fJDLName),
bb885a9e 172 fMergeExcludes(other.fMergeExcludes),
f965131e 173 fIncludePath(other.fIncludePath),
bb885a9e 174 fCloseSE(other.fCloseSE),
0df6ccf2 175 fFriendChainName(other.fFriendChainName),
c6cb3634 176 fJobTag(other.fJobTag),
4e5c5506 177 fInputFiles(0),
178 fPackages(0)
c57f56b7 179{
180// Copy ctor.
181 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 182 fRunRange[0] = other.fRunRange[0];
183 fRunRange[1] = other.fRunRange[1];
c57f56b7 184 if (other.fInputFiles) {
185 fInputFiles = new TObjArray();
186 TIter next(other.fInputFiles);
187 TObject *obj;
188 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
189 fInputFiles->SetOwner();
190 }
4e5c5506 191 if (other.fPackages) {
192 fPackages = new TObjArray();
193 TIter next(other.fPackages);
194 TObject *obj;
195 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
196 fPackages->SetOwner();
197 }
c57f56b7 198}
199
200//______________________________________________________________________________
201AliAnalysisAlien::~AliAnalysisAlien()
202{
203// Destructor.
204 if (fGridJDL) delete fGridJDL;
205 if (fInputFiles) delete fInputFiles;
4e5c5506 206 if (fPackages) delete fPackages;
c57f56b7 207}
208
209//______________________________________________________________________________
210AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
211{
212// Assignment.
213 if (this != &other) {
214 AliAnalysisGrid::operator=(other);
215 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
216 fPrice = other.fPrice;
217 fTTL = other.fTTL;
218 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
219 fMaxInitFailed = other.fMaxInitFailed;
220 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 221 fNtestFiles = other.fNtestFiles;
a3e84053 222 fNrunsPerMaster = other.fNrunsPerMaster;
223 fMaxMergeFiles = other.fMaxMergeFiles;
224 fNsubmitted = other.fNsubmitted;
225 fProductionMode = other.fProductionMode;
c57f56b7 226 fRunNumbers = other.fRunNumbers;
227 fExecutable = other.fExecutable;
0a1c1f7f 228 fExecutableCommand = other.fExecutableCommand;
c57f56b7 229 fArguments = other.fArguments;
230 fAnalysisMacro = other.fAnalysisMacro;
231 fAnalysisSource = other.fAnalysisSource;
232 fAdditionalLibs = other.fAdditionalLibs;
233 fSplitMode = other.fSplitMode;
234 fAPIVersion = other.fAPIVersion;
235 fROOTVersion = other.fROOTVersion;
236 fAliROOTVersion = other.fAliROOTVersion;
237 fUser = other.fUser;
238 fGridWorkingDir = other.fGridWorkingDir;
239 fGridDataDir = other.fGridDataDir;
240 fDataPattern = other.fDataPattern;
241 fGridOutputDir = other.fGridOutputDir;
242 fOutputArchive = other.fOutputArchive;
243 fOutputFiles = other.fOutputFiles;
244 fInputFormat = other.fInputFormat;
e7c71df0 245 fDatasetName = other.fDatasetName;
c57f56b7 246 fJDLName = other.fJDLName;
bb885a9e 247 fMergeExcludes = other.fMergeExcludes;
f965131e 248 fIncludePath = other.fIncludePath;
bb885a9e 249 fCloseSE = other.fCloseSE;
0df6ccf2 250 fFriendChainName = other.fFriendChainName;
c6cb3634 251 fJobTag = other.fJobTag;
c57f56b7 252 if (other.fInputFiles) {
253 fInputFiles = new TObjArray();
254 TIter next(other.fInputFiles);
255 TObject *obj;
256 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
257 fInputFiles->SetOwner();
258 }
4e5c5506 259 if (other.fPackages) {
260 fPackages = new TObjArray();
261 TIter next(other.fPackages);
262 TObject *obj;
263 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
264 fPackages->SetOwner();
265 }
c57f56b7 266 }
267 return *this;
268}
269
f965131e 270//______________________________________________________________________________
271void AliAnalysisAlien::AddIncludePath(const char *path)
272{
273// Add include path in the remote analysis macro.
274 TString p(path);
275 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
276 else fIncludePath += Form("-I%s ", path);
277}
278
c57f56b7 279//______________________________________________________________________________
280void AliAnalysisAlien::AddRunNumber(Int_t run)
281{
282// Add a run number to the list of runs to be processed.
283 if (fRunNumbers.Length()) fRunNumbers += " ";
284 fRunNumbers += Form("%d", run);
285}
286
ee75cfc3 287//______________________________________________________________________________
288void AliAnalysisAlien::AddRunNumber(const char* run)
289{
290// Add a run number to the list of runs to be processed.
291 if (fRunNumbers.Length()) fRunNumbers += " ";
292 fRunNumbers += run;
293}
294
c57f56b7 295//______________________________________________________________________________
296void AliAnalysisAlien::AddDataFile(const char *lfn)
297{
298// Adds a data file to the input to be analysed. The file should be a valid LFN
299// or point to an existing file in the alien workdir.
300 if (!fInputFiles) fInputFiles = new TObjArray();
301 fInputFiles->Add(new TObjString(lfn));
302}
303
304//______________________________________________________________________________
305Bool_t AliAnalysisAlien::Connect()
306{
307// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
308 if (gGrid && gGrid->IsConnected()) return kTRUE;
309 if (!gSystem->Getenv("alien_API_USER")) {
310 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
311 gSystem->Getenv("UID"));
312 return kFALSE;
313 }
314 if (!gGrid) {
315 Info("Connect", "Trying to connect to AliEn ...");
316 TGrid::Connect("alien://");
317 }
318 if (!gGrid || !gGrid->IsConnected()) {
319 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
320 return kFALSE;
321 }
322 fUser = gGrid->GetUser();
323 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
324 return kTRUE;
325}
326
327//______________________________________________________________________________
328void AliAnalysisAlien::CdWork()
329{
330// Check validity of alien workspace. Create directory if possible.
331 if (!Connect()) {
332 Error("CdWork", "Alien connection required");
333 return;
334 }
335 TString homedir = gGrid->GetHomeDirectory();
336 TString workdir = homedir + fGridWorkingDir;
923e2ca5 337 if (DirectoryExists(workdir)) {
338 gGrid->Cd(workdir);
339 return;
340 }
341 // Work directory not existing - create it
342 gGrid->Cd(homedir);
343 if (gGrid->Mkdir(workdir)) {
344 gGrid->Cd(fGridWorkingDir);
345 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
346 } else {
347 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
348 workdir.Data(), homedir.Data());
349 fGridWorkingDir = "";
350 }
c57f56b7 351}
352
353//______________________________________________________________________________
354Bool_t AliAnalysisAlien::CheckInputData()
355{
356// Check validity of input data. If necessary, create xml files.
a8739e8a 357 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 358 if (!fGridDataDir.Length()) {
359 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
360 return kFALSE;
361 }
362 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
363 return kTRUE;
c57f56b7 364 }
365 // Process declared files
366 Bool_t is_collection = kFALSE;
367 Bool_t is_xml = kFALSE;
368 Bool_t use_tags = kFALSE;
369 Bool_t checked = kFALSE;
370 CdWork();
371 TString file;
372 TString workdir = gGrid->GetHomeDirectory();
373 workdir += fGridWorkingDir;
374 if (fInputFiles) {
375 TObjString *objstr;
376 TIter next(fInputFiles);
377 while ((objstr=(TObjString*)next())) {
378 file = workdir;
379 file += "/";
380 file += objstr->GetString();
381 // Store full lfn path
382 if (FileExists(file)) objstr->SetString(file);
383 else {
384 file = objstr->GetName();
385 if (!FileExists(objstr->GetName())) {
386 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
387 objstr->GetName(), workdir.Data());
388 return kFALSE;
389 }
390 }
391 Bool_t iscoll, isxml, usetags;
392 CheckDataType(file, iscoll, isxml, usetags);
393 if (!checked) {
394 checked = kTRUE;
395 is_collection = iscoll;
396 is_xml = isxml;
397 use_tags = usetags;
398 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
399 } else {
400 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
401 Error("CheckInputData", "Some conflict was found in the types of inputs");
402 return kFALSE;
403 }
404 }
405 }
406 }
407 // Process requested run numbers
a8739e8a 408 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 409 // Check validity of alien data directory
410 if (!fGridDataDir.Length()) {
411 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
412 return kFALSE;
413 }
923e2ca5 414 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 415 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
416 return kFALSE;
417 }
418 if (is_collection) {
419 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
420 return kFALSE;
421 }
422
423 if (checked && !is_xml) {
424 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
425 return kFALSE;
426 }
427 // Check validity of run number(s)
428 TObjArray *arr;
429 TObjString *os;
319593fb 430 Int_t nruns = 0;
431 TString schunk;
c57f56b7 432 TString path;
433 if (!checked) {
434 checked = kTRUE;
435 use_tags = fDataPattern.Contains("tag");
436 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
437 }
438 if (use_tags != fDataPattern.Contains("tag")) {
439 Error("CheckInputData", "Cannot mix input files using/not using tags");
440 return kFALSE;
441 }
442 if (fRunNumbers.Length()) {
a8739e8a 443 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 444 arr = fRunNumbers.Tokenize(" ");
445 TIter next(arr);
446 while ((os=(TObjString*)next())) {
447 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 448 if (!DirectoryExists(path)) {
449 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 450 continue;
c57f56b7 451 }
452 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
453 TString msg = "\n##### file: ";
454 msg += path;
455 msg += " type: xml_collection;";
456 if (use_tags) msg += " using_tags: Yes";
457 else msg += " using_tags: No";
458 Info("CheckDataType", msg.Data());
319593fb 459 if (fNrunsPerMaster<2) {
d2a409b2 460 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 461 } else {
462 nruns++;
463 if (((nruns-1)%fNrunsPerMaster) == 0) {
464 schunk = os->GetString();
465 }
466 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
467 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 468 AddDataFile(schunk);
319593fb 469 }
c57f56b7 470 }
471 delete arr;
a8739e8a 472 } else {
473 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
474 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
475 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 476 if (!DirectoryExists(path)) {
477// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 478 continue;
479 }
480 path = Form("%s/%d.xml", workdir.Data(),irun);
481 TString msg = "\n##### file: ";
482 msg += path;
483 msg += " type: xml_collection;";
484 if (use_tags) msg += " using_tags: Yes";
485 else msg += " using_tags: No";
486 Info("CheckDataType", msg.Data());
319593fb 487 if (fNrunsPerMaster<2) {
d2a409b2 488 AddDataFile(Form("%d.xml",irun));
319593fb 489 } else {
490 nruns++;
491 if (((nruns-1)%fNrunsPerMaster) == 0) {
492 schunk = Form("%d", irun);
493 }
494 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
495 schunk += Form("_%d.xml", irun);
d2a409b2 496 AddDataFile(schunk);
319593fb 497 }
a8739e8a 498 }
c57f56b7 499 }
500 return kTRUE;
501}
502
503//______________________________________________________________________________
504Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
505{
506// Create dataset for the grid data directory + run number.
507 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
508 if (!Connect()) {
509 Error("CreateDataset", "Cannot create dataset with no grid connection");
510 return kFALSE;
511 }
512
513 // Cd workspace
514 CdWork();
515 TString workdir = gGrid->GetHomeDirectory();
516 workdir += fGridWorkingDir;
517
518 // Compose the 'find' command arguments
519 TString command;
520 TString options = "-x collection ";
bb885a9e 521 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 522 TString conditions = "";
523
524 TString file;
525 TString path;
319593fb 526 Int_t nruns = 0;
527 TString schunk;
ab254fd1 528 TGridCollection *cbase=0, *cadd=0;
d2a409b2 529 if (!fRunNumbers.Length() && !fRunRange[0]) {
530 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
531 // Make a single data collection from data directory.
532 path = fGridDataDir;
923e2ca5 533 if (!DirectoryExists(path)) {
d2a409b2 534 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
535 return kFALSE;
536 }
923e2ca5 537// CdWork();
d2a409b2 538 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
539 else file = Form("%s.xml", gSystem->BaseName(path));
14d25f30 540 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
d2a409b2 541 command = "find ";
542 command += options;
543 command += path;
544 command += " ";
545 command += pattern;
546 command += conditions;
84fcd93f 547 printf("command: %s\n", command.Data());
d2a409b2 548 TGridResult *res = gGrid->Command(command);
549 if (res) delete res;
550 // Write standard output to file
551 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
552 }
553 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
554 // Copy xml file to alien space
555 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
556 if (!FileExists(file)) {
557 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
558 return kFALSE;
559 }
560 // Update list of files to be processed.
561 }
562 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
563 return kTRUE;
564 }
c57f56b7 565 // Several runs
a8739e8a 566 if (fRunNumbers.Length()) {
567 TObjArray *arr = fRunNumbers.Tokenize(" ");
568 TObjString *os;
569 TIter next(arr);
570 while ((os=(TObjString*)next())) {
571 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 572 if (!DirectoryExists(path)) continue;
573// CdWork();
a8739e8a 574 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
575 else file = Form("%s.xml", os->GetString().Data());
319593fb 576 // If local collection file does not exist, create it via 'find' command.
577 if (gSystem->AccessPathName(file)) {
578 command = "find ";
579 command += options;
580 command += path;
581 command += pattern;
582 command += conditions;
583 TGridResult *res = gGrid->Command(command);
584 if (res) delete res;
585 // Write standard output to file
586 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
587 }
a8739e8a 588 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 589 // Check if there is one run per master job.
590 if (fNrunsPerMaster<2) {
591 if (FileExists(file)) {
592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
593 continue;
594 }
595 // Copy xml file to alien space
596 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
597 if (!FileExists(file)) {
598 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
599 delete arr;
600 return kFALSE;
601 }
602 } else {
603 nruns++;
604 if (((nruns-1)%fNrunsPerMaster) == 0) {
605 schunk = os->GetString();
606 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
607 } else {
608 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
84fcd93f 609 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
319593fb 610 cbase->Add(cadd);
611 delete cadd;
612 }
613 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
614 continue;
615 }
616 schunk += Form("_%s.xml", os->GetString().Data());
617 if (FileExists(schunk)) {
618 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
619 continue;
620 }
84fcd93f 621 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
319593fb 622 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
e95434bc 623 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
319593fb 624 if (!FileExists(schunk)) {
625 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
626 delete arr;
627 return kFALSE;
628 }
629 }
a8739e8a 630 }
631 delete arr;
632 } else {
633 // Process a full run range.
634 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
635 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 636 if (!DirectoryExists(path)) continue;
637// CdWork();
a8739e8a 638 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
639 else file = Form("%d.xml", irun);
319593fb 640 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 641 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
642// gGrid->Rm(file);
643 continue;
644 }
319593fb 645 // If local collection file does not exist, create it via 'find' command.
646 if (gSystem->AccessPathName(file)) {
647 command = "find ";
648 command += options;
649 command += path;
650 command += pattern;
651 command += conditions;
652 TGridResult *res = gGrid->Command(command);
653 if (res) delete res;
654 // Write standard output to file
655 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
656 }
a8739e8a 657 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 658 // Check if there is one run per master job.
659 if (fNrunsPerMaster<2) {
660 if (FileExists(file)) {
661 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
662 continue;
663 }
664 // Copy xml file to alien space
665 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
666 if (!FileExists(file)) {
667 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
668 return kFALSE;
669 }
670 } else {
671 nruns++;
95e5b448 672 // Check if the collection for the chunk exist locally.
673 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
674 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
84fcd93f 675 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
319593fb 676 if (((nruns-1)%fNrunsPerMaster) == 0) {
677 schunk = Form("%d", irun);
678 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
679 } else {
680 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
681 cbase->Add(cadd);
682 delete cadd;
683 }
684 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
685 continue;
686 }
687 schunk += Form("_%d.xml", irun);
688 if (FileExists(schunk)) {
689 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
690 continue;
691 }
84fcd93f 692 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
319593fb 693 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 694 if (FileExists(schunk)) {
695 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
696 continue;
697 }
319593fb 698 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
699 if (!FileExists(schunk)) {
700 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
701 return kFALSE;
702 }
703 }
c57f56b7 704 }
a8739e8a 705 }
c57f56b7 706 return kTRUE;
707}
708
709//______________________________________________________________________________
710Bool_t AliAnalysisAlien::CreateJDL()
711{
712// Generate a JDL file according to current settings. The name of the file is
713// specified by fJDLName.
714 Bool_t error = kFALSE;
715 TObjArray *arr = 0;
716 Bool_t copy = kTRUE;
717 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
718 Bool_t generate = kTRUE;
719 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
720 if (!Connect()) {
721 Error("CreateJDL", "Alien connection required");
722 return kFALSE;
723 }
724 // Check validity of alien workspace
725 CdWork();
726 TString workdir = gGrid->GetHomeDirectory();
727 workdir += fGridWorkingDir;
728 if (generate) {
729 TObjString *os;
730 if (!fInputFiles) {
731 Error("CreateJDL()", "Define some input files for your analysis.");
732 error = kTRUE;
733 }
734 // Compose list of input files
735 // Check if output files were defined
736 if (!fOutputFiles.Length()) {
737 Error("CreateJDL", "You must define at least one output file");
738 error = kTRUE;
739 }
740 // Check if an output directory was defined and valid
741 if (!fGridOutputDir.Length()) {
742 Error("CreateJDL", "You must define AliEn output directory");
743 error = kTRUE;
744 } else {
d2a409b2 745 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 746 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 747 if (gGrid->Mkdir(fGridOutputDir)) {
748 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
749 } else {
750 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 751 // error = kTRUE;
c57f56b7 752 }
753 }
754 gGrid->Cd(workdir);
755 }
756 // Exit if any error up to now
757 if (error) return kFALSE;
758 // Set JDL fields
759 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 760 fGridJDL->SetExecutable(fExecutable);
c57f56b7 761// fGridJDL->SetTTL((UInt_t)fTTL);
762 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
763 if (fMaxInitFailed > 0)
764 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
765 if (fSplitMaxInputFileNumber > 0)
766 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
767 if (fSplitMode.Length())
768 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
769// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
770 if (fAliROOTVersion.Length())
771 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
772 if (fROOTVersion.Length())
773 fGridJDL->AddToPackages("ROOT", fROOTVersion);
774 if (fAPIVersion.Length())
775 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
776 fGridJDL->SetInputDataListFormat(fInputFormat);
777 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 778 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 779 TString analysisFile = fExecutable;
780 analysisFile.ReplaceAll(".sh", ".root");
781 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 782 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
783 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
784 if (fAdditionalLibs.Length()) {
785 arr = fAdditionalLibs.Tokenize(" ");
786 TIter next(arr);
787 while ((os=(TObjString*)next())) {
788 if (os->GetString().Contains(".so")) continue;
789 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
790 }
791 delete arr;
792 }
4e5c5506 793 if (fPackages) {
794 TIter next(fPackages);
795 TObject *obj;
796 while ((obj=next()))
797 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
798 }
c57f56b7 799 if (fOutputArchive.Length()) {
800 arr = fOutputArchive.Tokenize(" ");
801 TIter next(arr);
802 while ((os=(TObjString*)next()))
43da816a 803 if (!os->GetString().Contains("@") && fCloseSE.Length())
804 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
805 else
806 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 807 delete arr;
808 }
c57f56b7 809 arr = fOutputFiles.Tokenize(" ");
810 TIter next(arr);
43da816a 811 while ((os=(TObjString*)next())) {
e1eaf596 812 // Ignore ouputs in jdl that are also in outputarchive
813 TString sout = os->GetString();
814 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
815 if (fOutputArchive.Contains(sout)) continue;
43da816a 816 if (!os->GetString().Contains("@") && fCloseSE.Length())
817 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
818 else
819 fGridJDL->AddToOutputSandbox(os->GetString());
820 }
c57f56b7 821 delete arr;
822// fGridJDL->SetPrice((UInt_t)fPrice);
823 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 824 TString validationScript = fExecutable;
825 validationScript.ReplaceAll(".sh", "_validation.sh");
826 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 827 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 828 // Write a jdl with 2 input parameters: collection name and output dir name.
829 WriteJDL(copy);
c57f56b7 830 }
831 // Copy jdl to grid workspace
a8739e8a 832 if (copy) {
c57f56b7 833 if (fAdditionalLibs.Length()) {
e7c71df0 834 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 835 TObjString *os;
836 TIter next(arr);
837 while ((os=(TObjString*)next())) {
c57f56b7 838 if (os->GetString().Contains(".so")) continue;
4e5c5506 839 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 840 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
841 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
842 }
843 delete arr;
844 }
4e5c5506 845 if (fPackages) {
846 TIter next(fPackages);
847 TObject *obj;
848 while ((obj=next())) {
849 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
850 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
851 }
852 }
c57f56b7 853 }
854 return kTRUE;
855}
856
a8739e8a 857//______________________________________________________________________________
d2a409b2 858Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 859{
860// Writes one or more JDL's corresponding to findex. If findex is negative,
861// all run numbers are considered in one go (jdl). For non-negative indices
862// they correspond to the indices in the array fInputFiles.
863 if (!fInputFiles) return kFALSE;
864 TObjString *os;
a8739e8a 865 TString workdir = gGrid->GetHomeDirectory();
866 workdir += fGridWorkingDir;
d2a409b2 867
868 if (!fRunNumbers.Length() && !fRunRange[0]) {
869 // One jdl with no parameters in case input data is specified by name.
a8739e8a 870 TIter next(fInputFiles);
871 while ((os=(TObjString*)next()))
872 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 873 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 874 } else {
d2a409b2 875 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
876 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 877 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 878 }
879
880
881 // Generate the JDL as a string
882 TString sjdl = fGridJDL->Generate();
883 Int_t index;
884 index = sjdl.Index("Executable");
885 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
886 index = sjdl.Index("Split ");
887 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
888 index = sjdl.Index("SplitMaxInputFileNumber");
889 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
890 index = sjdl.Index("InputDataCollection");
891 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 892 index = sjdl.Index("InputFile");
893 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
894 index = sjdl.Index("InputDataList ");
895 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
896 index = sjdl.Index("InputDataListFormat");
897 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
898 index = sjdl.Index("Price");
899 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
900 index = sjdl.Index("Requirements");
901 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
902 index = sjdl.Index("Packages");
903 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
904 index = sjdl.Index("User =");
905 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
906 index = sjdl.Index("TTL");
907 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
908 index = sjdl.Index("OutputFile");
909 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
910 index = sjdl.Index("OutputDir");
911 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
912 index = sjdl.Index("OutputArchive");
913 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
914 index = sjdl.Index("MaxInitFailed");
915 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
916 index = sjdl.Index("MasterResubmitThreshold");
917 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
918 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
919 index = sjdl.Index("Validationcommand");
920 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
921 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
922 sjdl.ReplaceAll("(member", "\n (member");
923 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
924 sjdl.ReplaceAll("{", "{\n ");
925 sjdl.ReplaceAll("};", "\n};");
926 sjdl.ReplaceAll("{\n \n", "{\n");
927 sjdl.ReplaceAll("\n\n", "\n");
928 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
929 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 930 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 931 index = sjdl.Index("JDLVariables");
932 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
933 // Write jdl to file
a8739e8a 934 ofstream out;
d2a409b2 935 out.open(fJDLName.Data(), ios::out);
a8739e8a 936 if (out.bad()) {
d2a409b2 937 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 938 return kFALSE;
939 }
940 out << sjdl << endl;
941
942 // Copy jdl to grid workspace
943 if (!copy) {
944 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
945 } else {
d2a409b2 946 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
947 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 948 if (fProductionMode)
b5fe9cba 949 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 950 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 951 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 952 }
953 return kTRUE;
954}
955
c57f56b7 956//______________________________________________________________________________
5513444a 957Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 958{
959// Returns true if file exists.
5513444a 960 if (!gGrid) return kFALSE;
c57f56b7 961 TGridResult *res = gGrid->Ls(lfn);
962 if (!res) return kFALSE;
963 TMap *map = dynamic_cast<TMap*>(res->At(0));
964 if (!map) {
965 delete res;
966 return kFALSE;
967 }
968 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
969 if (!objs || !objs->GetString().Length()) {
970 delete res;
971 return kFALSE;
972 }
973 delete res;
974 return kTRUE;
975}
976
923e2ca5 977//______________________________________________________________________________
978Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
979{
980// Returns true if directory exists. Can be also a path.
981 if (!gGrid) return kFALSE;
982 // Check if dirname is a path
983 TString dirstripped = dirname;
984 dirstripped = dirstripped.Strip();
985 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
986 TString dir = gSystem->BaseName(dirstripped);
987 dir += "/";
988 TString path = gSystem->DirName(dirstripped);
989 TGridResult *res = gGrid->Ls(path, "-F");
990 if (!res) return kFALSE;
991 TIter next(res);
992 TMap *map;
993 TObject *obj;
994 while ((map=dynamic_cast<TMap*>(next()))) {
995 obj = map->GetValue("name");
996 if (!obj) break;
997 if (dir == obj->GetName()) {
998 delete res;
999 return kTRUE;
1000 }
1001 }
1002 delete res;
1003 return kFALSE;
1004}
1005
c57f56b7 1006//______________________________________________________________________________
1007void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1008{
1009// Check input data type.
1010 is_collection = kFALSE;
1011 is_xml = kFALSE;
1012 use_tags = kFALSE;
1013 if (!gGrid) {
1014 Error("CheckDataType", "No connection to grid");
1015 return;
1016 }
1017 is_collection = IsCollection(lfn);
1018 TString msg = "\n##### file: ";
1019 msg += lfn;
1020 if (is_collection) {
1021 msg += " type: raw_collection;";
1022 // special treatment for collections
1023 is_xml = kFALSE;
1024 // check for tag files in the collection
1025 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1026 if (!res) {
1027 msg += " using_tags: No (unknown)";
1028 Info("CheckDataType", msg.Data());
1029 return;
1030 }
1031 const char* typeStr = res->GetKey(0, "origLFN");
1032 if (!typeStr || !strlen(typeStr)) {
1033 msg += " using_tags: No (unknown)";
1034 Info("CheckDataType", msg.Data());
1035 return;
1036 }
1037 TString file = typeStr;
1038 use_tags = file.Contains(".tag");
1039 if (use_tags) msg += " using_tags: Yes";
1040 else msg += " using_tags: No";
1041 Info("CheckDataType", msg.Data());
1042 return;
1043 }
1044 TString slfn(lfn);
1045 slfn.ToLower();
1046 is_xml = slfn.Contains(".xml");
1047 if (is_xml) {
1048 // Open xml collection and check if there are tag files inside
1049 msg += " type: xml_collection;";
1050 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1051 if (!coll) {
1052 msg += " using_tags: No (unknown)";
1053 Info("CheckDataType", msg.Data());
1054 return;
1055 }
1056 TMap *map = coll->Next();
1057 if (!map) {
1058 msg += " using_tags: No (unknown)";
1059 Info("CheckDataType", msg.Data());
1060 return;
1061 }
1062 map = (TMap*)map->GetValue("");
1063 TString file;
1064 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1065 use_tags = file.Contains(".tag");
1066 delete coll;
1067 if (use_tags) msg += " using_tags: Yes";
1068 else msg += " using_tags: No";
1069 Info("CheckDataType", msg.Data());
1070 return;
1071 }
1072 use_tags = slfn.Contains(".tag");
1073 if (slfn.Contains(".root")) msg += " type: root file;";
1074 else msg += " type: unhnown file;";
1075 if (use_tags) msg += " using_tags: Yes";
1076 else msg += " using_tags: No";
1077 Info("CheckDataType", msg.Data());
1078}
1079
4e5c5506 1080//______________________________________________________________________________
1081void AliAnalysisAlien::EnablePackage(const char *package)
1082{
1083// Enables a par file supposed to exist in the current directory.
1084 TString pkg(package);
1085 pkg.ReplaceAll(".par", "");
1086 pkg += ".par";
1087 if (gSystem->AccessPathName(pkg)) {
1088 Error("EnablePackage", "Package %s not found", pkg.Data());
1089 return;
1090 }
1091 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1092 Info("EnablePackage", "AliEn plugin will use .par packages");
1093 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1094 if (!fPackages) {
1095 fPackages = new TObjArray();
1096 fPackages->SetOwner();
1097 }
1098 fPackages->Add(new TObjString(pkg));
1099}
1100
d2a409b2 1101//______________________________________________________________________________
1102const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1103{
1104// Get job status for all jobs with jobid>jobidstart.
1105 static char mstatus[20];
1106 mstatus[0] = '\0';
1107 nrunning = 0;
1108 nwaiting = 0;
1109 nerror = 0;
1110 ndone = 0;
1111 TGridJobStatusList *list = gGrid->Ps("");
1112 if (!list) return mstatus;
1113 Int_t nentries = list->GetSize();
1114 TGridJobStatus *status;
1115 Int_t pid;
1116 for (Int_t ijob=0; ijob<nentries; ijob++) {
1117 status = (TGridJobStatus *)list->At(ijob);
1118 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1119 if (pid<jobidstart) continue;
1120 if (pid == lastid) {
1121 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1122 }
1123 switch (status->GetStatus()) {
1124 case TGridJobStatus::kWAITING:
1125 nwaiting++; break;
1126 case TGridJobStatus::kRUNNING:
1127 nrunning++; break;
1128 case TGridJobStatus::kABORTED:
1129 case TGridJobStatus::kFAIL:
1130 case TGridJobStatus::kUNKNOWN:
1131 nerror++; break;
1132 case TGridJobStatus::kDONE:
1133 ndone++;
1134 }
1135 }
1136 list->Delete();
1137 delete list;
1138 return mstatus;
1139}
1140
c57f56b7 1141//______________________________________________________________________________
1142Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1143{
1144// Returns true if file is a collection. Functionality duplicated from
1145// TAlien::Type() because we don't want to directly depend on TAlien.
1146 if (!gGrid) {
1147 Error("IsCollection", "No connection to grid");
1148 return kFALSE;
1149 }
1150 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1151 if (!res) return kFALSE;
1152 const char* typeStr = res->GetKey(0, "type");
1153 if (!typeStr || !strlen(typeStr)) return kFALSE;
1154 if (!strcmp(typeStr, "collection")) return kTRUE;
1155 delete res;
1156 return kFALSE;
1157}
1158
16a4353c 1159//______________________________________________________________________________
1160void AliAnalysisAlien::Print(Option_t *) const
1161{
1162// Print current plugin settings.
84fcd93f 1163 printf("### AliEn analysis plugin current settings ###\n");
1164 printf("= Production mode:______________________________ %d\n", fProductionMode);
1165 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1166 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1167 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
16a4353c 1168 if (fUser.Length())
84fcd93f 1169 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1170 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1171 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1172 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1173 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1174 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
16a4353c 1175 if (fRunNumbers.Length())
84fcd93f 1176 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
16a4353c 1177 if (fRunRange[0])
84fcd93f 1178 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
16a4353c 1179 if (!fRunRange[0] && !fRunNumbers.Length()) {
1180 TIter next(fInputFiles);
1181 TObject *obj;
1182 TString list;
1183 while ((obj=next())) list += obj->GetName();
84fcd93f 1184 printf("= Input files to be processed: _________________ %s\n", list.Data());
16a4353c 1185 }
1186 if (TestBit(AliAnalysisGrid::kTest))
84fcd93f 1187 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1188 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1189 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1190 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1191 printf("=====================================================================\n");
1192 printf("= Job price: ___________________________________ %d\n", fPrice);
1193 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1194 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
16a4353c 1195 if (fMaxInitFailed>0)
84fcd93f 1196 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
16a4353c 1197 if (fMasterResubmitThreshold>0)
84fcd93f 1198 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1199 if (fNrunsPerMaster>0)
84fcd93f 1200 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1201 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1202 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
16a4353c 1203 if (fArguments.Length())
84fcd93f 1204 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1205 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1206 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1207 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1208 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
16a4353c 1209 if (fDatasetName)
84fcd93f 1210 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1211 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
16a4353c 1212 if (fIncludePath.Data())
84fcd93f 1213 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
16a4353c 1214 if (fCloseSE.Length())
84fcd93f 1215 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
16a4353c 1216 if (fFriendChainName.Length())
84fcd93f 1217 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
16a4353c 1218 if (fPackages) {
1219 TIter next(fPackages);
1220 TObject *obj;
1221 TString list;
1222 while ((obj=next())) list += obj->GetName();
84fcd93f 1223 printf("= Par files to be used: ________________________ %s\n", list.Data());
16a4353c 1224 }
1225}
1226
c57f56b7 1227//______________________________________________________________________________
1228void AliAnalysisAlien::SetDefaults()
1229{
1230// Set default values for everything. What cannot be filled will be left empty.
1231 if (fGridJDL) delete fGridJDL;
1232 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1233 fPrice = 1;
1234 fTTL = 30000;
1235 fSplitMaxInputFileNumber = 100;
1236 fMaxInitFailed = 0;
1237 fMasterResubmitThreshold = 0;
bb885a9e 1238 fNtestFiles = 10;
a8739e8a 1239 fRunRange[0] = 0;
1240 fRunRange[1] = 0;
319593fb 1241 fNrunsPerMaster = 1;
16a4353c 1242 fMaxMergeFiles = 100;
c57f56b7 1243 fRunNumbers = "";
1244 fExecutable = "analysis.sh";
0a1c1f7f 1245 fExecutableCommand = "root -b -q";
c57f56b7 1246 fArguments = "";
1247 fAnalysisMacro = "myAnalysis.C";
1248 fAnalysisSource = "";
1249 fAdditionalLibs = "";
1250 fSplitMode = "se";
1251 fAPIVersion = "";
1252 fROOTVersion = "";
1253 fAliROOTVersion = "";
1254 fUser = ""; // Your alien user name
1255 fGridWorkingDir = "";
1256 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1257 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1258 fFriendChainName = "";
c57f56b7 1259 fGridOutputDir = "output";
1260 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1261 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1262 fInputFormat = "xml-single";
1263 fJDLName = "analysis.jdl";
c6cb3634 1264 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1265 fMergeExcludes = "";
c57f56b7 1266}
1267
1268//______________________________________________________________________________
1269Bool_t AliAnalysisAlien::MergeOutputs()
1270{
1271// Merge analysis outputs existing in the AliEn space.
1272 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1273 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1274 if (!Connect()) {
1275 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1276 return kFALSE;
1277 }
1278 // Get the output path
d2a409b2 1279 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1280 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1281 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1282 return kFALSE;
1283 }
1284 if (!fOutputFiles.Length()) {
1285 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1286 return kFALSE;
1287 }
1288 TObjArray *list = fOutputFiles.Tokenize(" ");
1289 TIter next(list);
1290 TObjString *str;
1291 TString command;
1292 TString output_file;
16a4353c 1293 TString output_chunk;
1294 TString previous_chunk;
1295 Int_t count_chunk = 0;
1296 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1297 Bool_t merged = kTRUE;
1298 while((str=(TObjString*)next())) {
1299 output_file = str->GetString();
1300 Int_t index = output_file.Index("@");
1301 if (index > 0) output_file.Remove(index);
319593fb 1302 // Skip already merged outputs
1303 if (!gSystem->AccessPathName(output_file)) {
1304 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1305 continue;
1306 }
bb885a9e 1307 if (fMergeExcludes.Length() &&
1308 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1309 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1310 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
84fcd93f 1311 printf("command: %s\n", command.Data());
c57f56b7 1312 TGridResult *res = gGrid->Command(command);
1313 if (!res) continue;
1314 TFileMerger *fm = 0;
1315 TIter nextmap(res);
ab254fd1 1316 TMap *map = 0;
16a4353c 1317 previous_chunk = "";
1318 count_chunk = 0;
319593fb 1319 // Check if there is a merge operation to resume
1320 output_chunk = output_file;
1321 output_chunk.ReplaceAll(".root", "_*.root");
1322 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1323 while (1) {
1324 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1325 if (!map) {
1326 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1327 delete res;
1328 return kFALSE;
1329 }
1330 output_chunk = output_file;
1331 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
84fcd93f 1332 printf("%s\n", output_chunk.Data());
319593fb 1333 count_chunk++;
1334 if (gSystem->AccessPathName(output_chunk)) continue;
1335 // Merged file with chunks up to <count_chunk> found
84fcd93f 1336 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
319593fb 1337 previous_chunk = output_chunk;
1338 break;
1339 }
1340 }
16a4353c 1341 count_zero = fMaxMergeFiles;
c57f56b7 1342 while ((map=(TMap*)nextmap())) {
16a4353c 1343 // Loop 'find' results and get next LFN
1344 if (count_zero == fMaxMergeFiles) {
1345 // First file in chunk - create file merger and add previous chunk if any.
1346 fm = new TFileMerger(kFALSE);
1347 fm->SetFastMethod(kTRUE);
1348 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1349 output_chunk = output_file;
1350 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1351 }
1352 // If last file found, put merged results in the output file
1353 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1354 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1355 if (!objs || !objs->GetString().Length()) {
16a4353c 1356 // Nothing found - skip this output
c57f56b7 1357 delete res;
16a4353c 1358 delete fm;
1359 break;
1360 }
1361 // Add file to be merged and decrement chunk counter.
1362 fm->AddFile(objs->GetString());
1363 count_zero--;
1364 if (count_zero==0 || map == res->Last()) {
1365 fm->OutputFile(output_chunk);
1366 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1367 // Nothing found - skip this output
1368 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1369 delete res;
1370 delete fm;
1371 break;
1372 }
1373 // Merge the outputs, then go to next chunk
1374 if (!fm->Merge()) {
1375 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1376 delete res;
1377 delete fm;
1378 merged = kFALSE;
1379 break;
1380 } else {
1381 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1382 gSystem->Unlink(previous_chunk);
16a4353c 1383 }
1384 if (map == res->Last()) {
1385 delete res;
1386 delete fm;
1387 break;
1388 }
1389 count_chunk++;
1390 count_zero = fMaxMergeFiles;
1391 previous_chunk = output_chunk;
c57f56b7 1392 }
c57f56b7 1393 }
c57f56b7 1394 }
1395 if (!merged) {
1396 Error("MergeOutputs", "Terminate() will NOT be executed");
1397 }
1398 return merged;
1399}
1400
bb885a9e 1401//______________________________________________________________________________
1402void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1403{
1404// Use the output files connected to output containers from the analysis manager
1405// rather than the files defined by SetOutputFiles
1406 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1407 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1408 analysis manager");
1409 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1410}
1411
5513444a 1412//______________________________________________________________________________
1413Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1414{
1415// Start remote grid analysis.
1416
43da816a 1417 // Check if output files have to be taken from the analysis manager
1418 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1419 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1420 if (!mgr || !mgr->IsInitialized()) {
1421 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1422 return kFALSE;
43da816a 1423 }
1424 fOutputFiles = "";
1425 TIter next(mgr->GetOutputs());
1426 AliAnalysisDataContainer *output;
1427 while ((output=(AliAnalysisDataContainer*)next())) {
1428 const char *filename = output->GetFileName();
1429 if (!(strcmp(filename, "default"))) {
1430 if (!mgr->GetOutputEventHandler()) continue;
1431 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1432 }
84fcd93f 1433 if (fOutputFiles.Contains(filename)) continue;
43da816a 1434 if (fOutputFiles.Length()) fOutputFiles += " ";
1435 fOutputFiles += filename;
1436 }
c07b9ce2 1437 // Add extra files registered to the analysis manager
1438 if (mgr->GetExtraFiles().Length()) {
1439 if (fOutputFiles.Length()) fOutputFiles += " ";
1440 fOutputFiles += mgr->GetExtraFiles();
84fcd93f 1441 }
43da816a 1442 }
f7b1cbc2 1443// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1444 if (TestBit(AliAnalysisGrid::kOffline)) {
1445 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1446 \n there nor any job run. You can revise the JDL and analysis \
1447 \n macro then run the same in \"submit\" mode.");
1448 } else if (TestBit(AliAnalysisGrid::kTest)) {
1449 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1450 \n dataset.");
1451 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1452 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1453 \n space and job submitted.");
1454 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1455 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1456 return kTRUE;
c57f56b7 1457 } else {
1458 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1459 }
1460
1461 if (!Connect()) {
1462 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1463 return kFALSE;
16a4353c 1464 }
1465 Print();
c57f56b7 1466 if (!CheckInputData()) {
1467 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1468 return kFALSE;
c57f56b7 1469 }
1470 CreateDataset(fDataPattern);
1471 WriteAnalysisFile();
1472 WriteAnalysisMacro();
1473 WriteExecutable();
1474 WriteValidationScript();
5513444a 1475 if (!CreateJDL()) return kFALSE;
1476 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1477 if (TestBit(AliAnalysisGrid::kTest)) {
1478 // Locally testing the analysis
1479 Info("StartAnalysis", "\n_______________________________________________________________________ \
1480 \n Running analysis script in a daughter shell as on a worker node \
1481 \n_______________________________________________________________________");
1482 TObjArray *list = fOutputFiles.Tokenize(" ");
1483 TIter next(list);
1484 TObjString *str;
1485 TString output_file;
1486 while((str=(TObjString*)next())) {
1487 output_file = str->GetString();
1488 Int_t index = output_file.Index("@");
1489 if (index > 0) output_file.Remove(index);
43da816a 1490 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1491 }
1492 delete list;
1493 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1494 TString validationScript = fExecutable;
1495 validationScript.ReplaceAll(".sh", "_validation.sh");
1496 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1497// gSystem->Exec("cat stdout");
5513444a 1498 return kFALSE;
c57f56b7 1499 }
5513444a 1500 // Check if submitting is managed by LPM manager
a3e84053 1501 if (fProductionMode) {
5513444a 1502 TString prodfile = fJDLName;
1503 prodfile.ReplaceAll(".jdl", ".prod");
1504 WriteProductionFile(prodfile);
1505 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1506 return kFALSE;
1507 }
a8739e8a 1508 // Submit AliEn job(s)
d2a409b2 1509 gGrid->Cd(fGridOutputDir);
a8739e8a 1510 TGridResult *res;
c57f56b7 1511 TString jobID = "";
d2a409b2 1512 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1513 // Submit a given xml or a set of runs
a8739e8a 1514 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
84fcd93f 1515 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
a8739e8a 1516 if (res) {
1517 const char *cjobId = res->GetKey(0,"jobId");
1518 if (!cjobId) {
1519 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1520 return kFALSE;
a8739e8a 1521 } else {
1522 Info("StartAnalysis", "\n_______________________________________________________________________ \
1523 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1524 \n_______________________________________________________________________",
1525 fJDLName.Data(), cjobId);
1526 jobID = cjobId;
1527 }
1528 delete res;
1529 }
1530 } else {
d2a409b2 1531 // Submit for a range of enumeration of runs.
1532 Submit();
c57f56b7 1533 }
a8739e8a 1534
c57f56b7 1535 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1536 \n You may exit at any time and terminate the job later using the option <terminate> \
1537 \n ##################################################################################", jobID.Data());
bb885a9e 1538 gSystem->Exec("aliensh");
5513444a 1539 return kTRUE;
c57f56b7 1540}
1541
d2a409b2 1542//______________________________________________________________________________
1543void AliAnalysisAlien::Submit()
1544{
1545// Submit all master jobs.
1546 Int_t nmasterjobs = fInputFiles->GetEntries();
1547 Long_t tshoot = gSystem->Now();
1548 if (!fNsubmitted) SubmitNext();
1549 while (fNsubmitted < nmasterjobs) {
1550 Long_t now = gSystem->Now();
1551 if ((now-tshoot)>30000) {
1552 tshoot = now;
1553 SubmitNext();
1554 }
1555 }
1556}
1557
1558//______________________________________________________________________________
1559void AliAnalysisAlien::SubmitNext()
1560{
1561// Submit next bunch of master jobs if the queue is free.
1562 static Bool_t iscalled = kFALSE;
1563 static Int_t firstmaster = 0;
1564 static Int_t lastmaster = 0;
1565 static Int_t npermaster = 0;
1566 if (iscalled) return;
1567 iscalled = kTRUE;
1568 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1569 Int_t ntosubmit = 0;
1570 TGridResult *res;
1571 TString jobID = "";
1572 if (!fNsubmitted) ntosubmit = 1;
1573 else {
1574 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
84fcd93f 1575 printf("=== master %d: %s\n", lastmaster, status.Data());
d2a409b2 1576 // If last master not split, just return
1577 if (status != "SPLIT") {iscalled = kFALSE; return;}
1578 // No more than 100 waiting jobs
1579 if (nwaiting>100) {iscalled = kFALSE; return;}
1580 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1581 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
84fcd93f 1582 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
d2a409b2 1583 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1584 }
1585 Int_t nmasterjobs = fInputFiles->GetEntries();
1586 for (Int_t i=0; i<ntosubmit; i++) {
1587 // Submit for a range of enumeration of runs.
1588 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1589 TString query;
1590 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
84fcd93f 1591 printf("********* %s\n",query.Data());
d2a409b2 1592 res = gGrid->Command(query);
1593 if (res) {
98ca124f 1594 TString cjobId1 = res->GetKey(0,"jobId");
1595 if (!cjobId1.Length()) {
d2a409b2 1596 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1597 iscalled = kFALSE;
1598 return;
1599 } else {
1600 Info("StartAnalysis", "\n_______________________________________________________________________ \
1601 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1602 \n_______________________________________________________________________",
98ca124f 1603 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1604 jobID += cjobId1;
1605 jobID += " ";
98ca124f 1606 lastmaster = cjobId1.Atoi();
d2a409b2 1607 if (!firstmaster) firstmaster = lastmaster;
1608 fNsubmitted++;
1609 }
1610 delete res;
1611 }
1612 }
1613 iscalled = kFALSE;
1614}
1615
c57f56b7 1616//______________________________________________________________________________
1617void AliAnalysisAlien::WriteAnalysisFile()
1618{
f10e8481 1619// Write current analysis manager into the file <analysisFile>
1620 TString analysisFile = fExecutable;
1621 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1622 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1623 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1624 if (!mgr || !mgr->IsInitialized()) {
1625 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1626 return;
1627 }
1628 // Check analysis type
1629 TObject *handler;
1630 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1631 handler = (TObject*)mgr->GetInputEventHandler();
1632 if (handler) {
1633 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1634 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1635 }
1636 TDirectory *cdir = gDirectory;
f10e8481 1637 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1638 if (file) {
1639 mgr->Write();
1640 delete file;
1641 }
1642 if (cdir) cdir->cd();
f10e8481 1643 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1644 }
1645 Bool_t copy = kTRUE;
1646 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1647 if (copy) {
1648 CdWork();
1649 TString workdir = gGrid->GetHomeDirectory();
1650 workdir += fGridWorkingDir;
f10e8481 1651 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1652 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1653 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1654 }
1655}
1656
1657//______________________________________________________________________________
1658void AliAnalysisAlien::WriteAnalysisMacro()
1659{
1660// Write the analysis macro that will steer the analysis in grid mode.
1661 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1662 ofstream out;
1663 out.open(fAnalysisMacro.Data(), ios::out);
1664 if (!out.good()) {
1665 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1666 return;
1667 }
1668 TString func = fAnalysisMacro;
1669 TString type = "ESD";
1670 TString comment = "// Analysis using ";
1671 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1672 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1673 type = "AOD";
1674 comment += "AOD";
1675 }
0df6ccf2 1676 if (type!="AOD" && fFriendChainName!="") {
1677 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1678 return;
1679 }
c57f56b7 1680 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1681 else comment += " data";
1682 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1683 func.ReplaceAll(".C", "");
1684 out << "void " << func.Data() << "()" << endl;
1685 out << "{" << endl;
1686 out << comment.Data() << endl;
1687 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1688 out << " TStopwatch timer;" << endl;
1689 out << " timer.Start();" << endl << endl;
c57f56b7 1690 out << "// load base root libraries" << endl;
1691 out << " gSystem->Load(\"libTree\");" << endl;
1692 out << " gSystem->Load(\"libGeom\");" << endl;
1693 out << " gSystem->Load(\"libVMC\");" << endl;
1694 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1695 out << "// Load analysis framework libraries" << endl;
4e5c5506 1696 if (!fPackages) {
4e5c5506 1697 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1698 out << " gSystem->Load(\"libESD\");" << endl;
1699 out << " gSystem->Load(\"libAOD\");" << endl;
1700 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1701 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1702 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1703 } else {
4e5c5506 1704 TIter next(fPackages);
1705 TObject *obj;
57377eb5 1706 TString pkgname;
1707 Bool_t hasSTEERBase = kFALSE;
1708 Bool_t hasESD = kFALSE;
1709 Bool_t hasAOD = kFALSE;
1710 Bool_t hasANALYSIS = kFALSE;
1711 Bool_t hasANALYSISalice = kFALSE;
1712 Bool_t hasCORRFW = kFALSE;
1713 while ((obj=next())) {
1714 pkgname = obj->GetName();
4478e6f1 1715 if (pkgname == "STEERBase" ||
1716 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1717 if (pkgname == "ESD" ||
1718 pkgname == "ESD.par") hasESD = kTRUE;
1719 if (pkgname == "AOD" ||
1720 pkgname == "AOD.par") hasAOD = kTRUE;
1721 if (pkgname == "ANALYSIS" ||
1722 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1723 if (pkgname == "ANALYSISalice" ||
1724 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1725 if (pkgname == "CORRFW" ||
1726 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1727 }
1728 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1729 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1730 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1731 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1732 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1733 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1734 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1735 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1736 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1737 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1738 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1739 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1740 out << "// Compile other par packages" << endl;
1741 next.Reset();
fcc9bb6f 1742 while ((obj=next())) {
1743 pkgname = obj->GetName();
4478e6f1 1744 if (pkgname == "STEERBase" ||
1745 pkgname == "STEERBase.par" ||
1746 pkgname == "ESD" ||
1747 pkgname == "ESD.par" ||
1748 pkgname == "AOD" ||
1749 pkgname == "AOD.par" ||
1750 pkgname == "ANALYSIS" ||
1751 pkgname == "ANALYSIS.par" ||
1752 pkgname == "ANALYSISalice" ||
1753 pkgname == "ANALYSISalice.par" ||
1754 pkgname == "CORRFW" ||
1755 pkgname == "CORRFW.par") continue;
4e5c5506 1756 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1757 }
4e5c5506 1758 }
43da816a 1759 out << "// include path" << endl;
1760 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1761 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1762 if (fAdditionalLibs.Length()) {
1763 out << "// Add aditional AliRoot libraries" << endl;
1764 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1765 TIter next(list);
1766 TObjString *str;
1767 while((str=(TObjString*)next())) {
1768 if (str->GetString().Contains(".so"))
1769 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1770 }
1771 if (list) delete list;
1772 }
1773 out << endl;
c57f56b7 1774 out << "// analysis source to be compiled at runtime (if any)" << endl;
1775 if (fAnalysisSource.Length()) {
1776 TObjArray *list = fAnalysisSource.Tokenize(" ");
1777 TIter next(list);
1778 TObjString *str;
1779 while((str=(TObjString*)next())) {
1780 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1781 }
1782 if (list) delete list;
1783 }
1784 out << endl;
1785 out << "// connect to AliEn and make the chain" << endl;
1786 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1787 if (IsUsingTags()) {
1788 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1789 } else {
76535c98 1790 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1791 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1792 } else {
1793 out << " // Check if the macro to create the chain was provided" << endl;
1794 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1795 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1796 out << " return;" << endl;
1797 out << " }" << endl;
1798 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1799 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1800 }
c57f56b7 1801 }
1802 out << "// read the analysis manager from file" << endl;
f10e8481 1803 TString analysisFile = fExecutable;
1804 analysisFile.ReplaceAll(".sh", ".root");
1805 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1806 out << " if (!file) return;" << endl;
1807 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1808 out << " AliAnalysisManager *mgr = 0;" << endl;
1809 out << " TKey *key;" << endl;
1810 out << " while ((key=(TKey*)nextkey())) {" << endl;
1811 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1812 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1813 out << " };" << endl;
1814 out << " if (!mgr) {" << endl;
f10e8481 1815 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1816 out << " return;" << endl;
1817 out << " }" << endl << endl;
1818 out << " mgr->PrintStatus();" << endl;
1819 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1820 out << " timer.Stop();" << endl;
1821 out << " timer.Print();" << endl;
c57f56b7 1822 out << "}" << endl << endl;
1823 if (IsUsingTags()) {
1824 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1825 out << "{" << endl;
1826 out << "// Create a chain using tags from the xml file." << endl;
1827 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1828 out << " if (!coll) {" << endl;
1829 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1830 out << " return NULL;" << endl;
1831 out << " }" << endl;
1832 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1833 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1834 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1835 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1836 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1837 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1838 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1839 out << " // Check if the cuts configuration file was provided" << endl;
1840 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1841 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1842 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1843 out << " }" << endl;
0df6ccf2 1844 if (fFriendChainName=="") {
1845 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1846 } else {
1847 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1848 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1849 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1850 }
c57f56b7 1851 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1852 out << " chain->ls();" << endl;
1853 out << " return chain;" << endl;
fcc9bb6f 1854 out << "}" << endl << endl;
c57f56b7 1855 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1856 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1857 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1858 msg += " AliLHCTagCuts *lhcCuts,\n";
1859 msg += " AliDetectorTagCuts *detCuts,\n";
1860 msg += " AliEventTagCuts *evCuts)";
1861 Info("WriteAnalysisMacro", msg.Data());
1862 }
0df6ccf2 1863 }
1864 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1865 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1866 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1867 out << "{" << endl;
1868 out << "// Create a chain using url's from xml file" << endl;
1869 out << " TString treename = type;" << endl;
1870 out << " treename.ToLower();" << endl;
1871 out << " treename += \"Tree\";" << endl;
e02fee64 1872 out << " printf(\"***************************************\\n\");" << endl;
1873 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1874 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1875 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1876 out << " if (!coll) {" << endl;
1877 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1878 out << " return NULL;" << endl;
1879 out << " }" << endl;
1880 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1881 if(fFriendChainName!="") {
1882 out << " TChain *chainFriend = new TChain(treename);" << endl;
1883 }
c57f56b7 1884 out << " coll->Reset();" << endl;
0df6ccf2 1885 out << " while (coll->Next()) {" << endl;
1886 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1887 if(fFriendChainName!="") {
1888 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1889 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1890 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1891 out << " chainFriend->Add(fileFriend.Data());" << endl;
1892 }
1893 out << " }" << endl;
c57f56b7 1894 out << " if (!chain->GetNtrees()) {" << endl;
1895 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1896 out << " return NULL;" << endl;
1897 out << " }" << endl;
0df6ccf2 1898 if(fFriendChainName!="") {
1899 out << " chain->AddFriend(chainFriend);" << endl;
1900 }
c57f56b7 1901 out << " return chain;" << endl;
fcc9bb6f 1902 out << "}" << endl << endl;
c57f56b7 1903 }
4e5c5506 1904 if (fPackages) {
fcc9bb6f 1905 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1906 out << "Bool_t SetupPar(const char *package) {" << endl;
1907 out << "// Compile the package and set it up." << endl;
1908 out << " TString pkgdir = package;" << endl;
1909 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1910 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1911 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1912 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1913 out << " // Check for BUILD.sh and execute" << endl;
1914 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1915 out << " printf(\"*******************************\\n\");" << endl;
1916 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1917 out << " printf(\"*******************************\\n\");" << endl;
1918 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1919 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1920 out << " gSystem->ChangeDirectory(cdir);" << endl;
1921 out << " return kFALSE;" << endl;
1922 out << " }" << endl;
1923 out << " } else {" << endl;
fcc9bb6f 1924 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1925 out << " gSystem->ChangeDirectory(cdir);" << endl;
1926 out << " return kFALSE;" << endl;
1927 out << " }" << endl;
1928 out << " // Check for SETUP.C and execute" << endl;
1929 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1930 out << " printf(\"*******************************\\n\");" << endl;
1931 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1932 out << " printf(\"*******************************\\n\");" << endl;
1933 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1934 out << " } else {" << endl;
fcc9bb6f 1935 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1936 out << " gSystem->ChangeDirectory(cdir);" << endl;
1937 out << " return kFALSE;" << endl;
1938 out << " }" << endl;
1939 out << " // Restore original workdir" << endl;
1940 out << " gSystem->ChangeDirectory(cdir);" << endl;
1941 out << " return kTRUE;" << endl;
1942 out << "}" << endl;
1943 }
c57f56b7 1944 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1945 }
1946 Bool_t copy = kTRUE;
1947 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1948 if (copy) {
1949 CdWork();
1950 TString workdir = gGrid->GetHomeDirectory();
1951 workdir += fGridWorkingDir;
1952 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1953 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1954 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1955 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1956 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1957 }
1958 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1959 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1960 }
1961}
1962
1963//______________________________________________________________________________
1964void AliAnalysisAlien::WriteExecutable()
1965{
1966// Generate the alien executable script.
1967 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1968 ofstream out;
1969 out.open(fExecutable.Data(), ios::out);
1970 if (out.bad()) {
5513444a 1971 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1972 return;
1973 }
1974 out << "#!/bin/bash" << endl;
c57f56b7 1975 out << "echo \"=========================================\"" << endl;
1976 out << "echo \"############## PATH : ##############\"" << endl;
1977 out << "echo $PATH" << endl;
1978 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1979 out << "echo $LD_LIBRARY_PATH" << endl;
1980 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1981 out << "echo $ROOTSYS" << endl;
1982 out << "echo \"############## which root : ##############\"" << endl;
1983 out << "which root" << endl;
1984 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1985 out << "echo $ALICE_ROOT" << endl;
1986 out << "echo \"############## which aliroot : ##############\"" << endl;
1987 out << "which aliroot" << endl;
1988 out << "echo \"=========================================\"" << endl << endl;
74a53467 1989 // Make sure we can properly compile par files
1990 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
0a1c1f7f 1991 out << fExecutableCommand << " ";
c57f56b7 1992 out << fAnalysisMacro.Data() << endl << endl;
1993 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1994 }
1995 Bool_t copy = kTRUE;
1996 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1997 if (copy) {
1998 CdWork();
1999 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 2000 TString bindir = Form("%s/bin", workdir.Data());
2001 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2002 workdir += fGridWorkingDir;
2003 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2004 if (FileExists(executable)) gGrid->Rm(executable);
2005 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2006 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2007 }
2008}
2009
2010//______________________________________________________________________________
5513444a 2011void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2012{
2013// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2014// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2015// Next lines: full_path_to_dataset XXX (XXX is a string)
2016// To submit, one has to: submit jdl XXX for all lines
2017 ofstream out;
2018 out.open(filename, ios::out);
2019 if (out.bad()) {
2020 Error("WriteProductionFile", "Bad file name: %s", filename);
2021 return;
2022 }
2023 TString workdir = gGrid->GetHomeDirectory();
2024 workdir += fGridWorkingDir;
f5e8c702 2025 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2026 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2027 out << locjdl << " " << njobspermaster << endl;
5513444a 2028 Int_t nmasterjobs = fInputFiles->GetEntries();
2029 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2030 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2031 }
2032 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2033 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2034}
2035
2036//______________________________________________________________________________
c57f56b7 2037void AliAnalysisAlien::WriteValidationScript()
2038{
2039// Generate the alien validation script.
2040 // Generate the validation script
2041 TObjString *os;
0d5d317c 2042 TString validationScript = fExecutable;
2043 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2044 if (!Connect()) {
2045 Error("WriteValidationScript", "Alien connection required");
2046 return;
2047 }
2048 TString out_stream = "";
2049 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2050 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2051 ofstream out;
0d5d317c 2052 out.open(validationScript, ios::out);
c57f56b7 2053 out << "#!/bin/bash" << endl;
2054 out << "##################################################" << endl;
2055 out << "validateout=`dirname $0`" << endl;
2056 out << "validatetime=`date`" << endl;
2057 out << "validated=\"0\";" << endl;
2058 out << "error=0" << endl;
2059 out << "if [ -z $validateout ]" << endl;
2060 out << "then" << endl;
2061 out << " validateout=\".\"" << endl;
2062 out << "fi" << endl << endl;
2063 out << "cd $validateout;" << endl;
2064 out << "validateworkdir=`pwd`;" << endl << endl;
2065 out << "echo \"*******************************************************\"" << out_stream << endl;
2066 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2067 out << "" << endl;
2068 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2069 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2070 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2071 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2072 out << "ls -la ./" << out_stream << endl;
2073 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2074 out << "##################################################" << endl;
ebec370a 2075
2076 out << "" << endl;
2077 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2078 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2079 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2080 out << "" << endl;
2081
2082 out << "if [ ! -f stderr ] ; then" << endl;
2083 out << " error=1" << endl;
2084 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2085 out << " echo \"Error = $error\" " << out_stream << endl;
2086 out << "fi" << endl;
2087
2088 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2089 out << " error=1" << endl;
2090 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2091 out << " echo \"$parArch\" " << out_stream << endl;
2092 out << " echo \"Error = $error\" " << out_stream << endl;
2093 out << "fi" << endl;
2094
2095 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2096 out << " error=1" << endl;
2097 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2098 out << " echo \"$segViol\" " << out_stream << endl;
2099 out << " echo \"Error = $error\" " << out_stream << endl;
2100 out << "fi" << endl;
2101
2102 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2103 out << " error=1" << endl;
2104 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2105 out << " echo \"$segFault\" " << out_stream << endl;
2106 out << " echo \"Error = $error\" " << out_stream << endl;
2107 out << "fi" << endl;
2108
2109 // Part dedicated to the specific analyses running into the train
2110
c57f56b7 2111 TObjArray *arr = fOutputFiles.Tokenize(" ");
2112 TIter next1(arr);
2113 TString output_file;
2114 while ((os=(TObjString*)next1())) {
2115 output_file = os->GetString();
2116 Int_t index = output_file.Index("@");
2117 if (index > 0) output_file.Remove(index);
2118 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2119 out << " error=1" << endl;
2120 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2121 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2122 out << "fi" << endl;
2123 }
2124 delete arr;
923e2ca5 2125 out << "if ! [ -f outputs_valid ] ; then" << endl;
2126 out << " error=1" << endl;
2127 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2128 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2129 out << "fi" << endl;
2130
c57f56b7 2131 out << "if [ $error = 0 ] ; then" << endl;
2132 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2133 out << "fi" << endl;
2134
2135 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2136 out << "echo \"*******************************************************\"" << out_stream << endl;
2137 out << "cd -" << endl;
2138 out << "exit $error" << endl;
2139 }
2140 Bool_t copy = kTRUE;
2141 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2142 if (copy) {
2143 CdWork();
2144 TString workdir = gGrid->GetHomeDirectory();
2145 workdir += fGridWorkingDir;
0d5d317c 2146 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2147 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2148 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2149 }
2150}