]> git.uio.no Git - u/mrichter/AliRoot.git/blame - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Added method SetExecutableCommand(const char *command) to the plugin. The default
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
0a1c1f7f 59 fExecutableCommand(),
c57f56b7 60 fArguments(),
61 fAnalysisMacro(),
62 fAnalysisSource(),
63 fAdditionalLibs(),
64 fSplitMode(),
65 fAPIVersion(),
66 fROOTVersion(),
67 fAliROOTVersion(),
68 fUser(),
69 fGridWorkingDir(),
70 fGridDataDir(),
71 fDataPattern(),
72 fGridOutputDir(),
73 fOutputArchive(),
74 fOutputFiles(),
75 fInputFormat(),
e7c71df0 76 fDatasetName(),
c57f56b7 77 fJDLName(),
bb885a9e 78 fMergeExcludes(),
f965131e 79 fIncludePath(),
bb885a9e 80 fCloseSE(),
0df6ccf2 81 fFriendChainName(),
c6cb3634 82 fJobTag(),
4e5c5506 83 fInputFiles(0),
84 fPackages(0)
c57f56b7 85{
86// Dummy ctor.
87 SetDefaults();
88}
89
90//______________________________________________________________________________
91AliAnalysisAlien::AliAnalysisAlien(const char *name)
92 :AliAnalysisGrid(name),
93 fGridJDL(NULL),
94 fPrice(0),
95 fTTL(0),
96 fSplitMaxInputFileNumber(0),
97 fMaxInitFailed(0),
98 fMasterResubmitThreshold(0),
bb885a9e 99 fNtestFiles(0),
319593fb 100 fNrunsPerMaster(0),
16a4353c 101 fMaxMergeFiles(0),
d2a409b2 102 fNsubmitted(0),
a3e84053 103 fProductionMode(0),
c57f56b7 104 fRunNumbers(),
105 fExecutable(),
0a1c1f7f 106 fExecutableCommand(),
c57f56b7 107 fArguments(),
108 fAnalysisMacro(),
109 fAnalysisSource(),
110 fAdditionalLibs(),
111 fSplitMode(),
112 fAPIVersion(),
113 fROOTVersion(),
114 fAliROOTVersion(),
115 fUser(),
116 fGridWorkingDir(),
117 fGridDataDir(),
118 fDataPattern(),
119 fGridOutputDir(),
120 fOutputArchive(),
121 fOutputFiles(),
122 fInputFormat(),
e7c71df0 123 fDatasetName(),
c57f56b7 124 fJDLName(),
bb885a9e 125 fMergeExcludes(),
f965131e 126 fIncludePath(),
bb885a9e 127 fCloseSE(),
0df6ccf2 128 fFriendChainName(),
c6cb3634 129 fJobTag(),
4e5c5506 130 fInputFiles(0),
131 fPackages(0)
c57f56b7 132{
133// Default ctor.
134 SetDefaults();
135}
136
137//______________________________________________________________________________
138AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
139 :AliAnalysisGrid(other),
140 fGridJDL(NULL),
141 fPrice(other.fPrice),
142 fTTL(other.fTTL),
143 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
144 fMaxInitFailed(other.fMaxInitFailed),
145 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 146 fNtestFiles(other.fNtestFiles),
319593fb 147 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 148 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 149 fNsubmitted(other.fNsubmitted),
a3e84053 150 fProductionMode(other.fProductionMode),
c57f56b7 151 fRunNumbers(other.fRunNumbers),
152 fExecutable(other.fExecutable),
0a1c1f7f 153 fExecutableCommand(other.fExecutableCommand),
c57f56b7 154 fArguments(other.fArguments),
155 fAnalysisMacro(other.fAnalysisMacro),
156 fAnalysisSource(other.fAnalysisSource),
157 fAdditionalLibs(other.fAdditionalLibs),
158 fSplitMode(other.fSplitMode),
159 fAPIVersion(other.fAPIVersion),
160 fROOTVersion(other.fROOTVersion),
161 fAliROOTVersion(other.fAliROOTVersion),
162 fUser(other.fUser),
163 fGridWorkingDir(other.fGridWorkingDir),
164 fGridDataDir(other.fGridDataDir),
165 fDataPattern(other.fDataPattern),
166 fGridOutputDir(other.fGridOutputDir),
167 fOutputArchive(other.fOutputArchive),
168 fOutputFiles(other.fOutputFiles),
169 fInputFormat(other.fInputFormat),
e7c71df0 170 fDatasetName(other.fDatasetName),
c57f56b7 171 fJDLName(other.fJDLName),
bb885a9e 172 fMergeExcludes(other.fMergeExcludes),
f965131e 173 fIncludePath(other.fIncludePath),
bb885a9e 174 fCloseSE(other.fCloseSE),
0df6ccf2 175 fFriendChainName(other.fFriendChainName),
c6cb3634 176 fJobTag(other.fJobTag),
4e5c5506 177 fInputFiles(0),
178 fPackages(0)
c57f56b7 179{
180// Copy ctor.
181 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 182 fRunRange[0] = other.fRunRange[0];
183 fRunRange[1] = other.fRunRange[1];
c57f56b7 184 if (other.fInputFiles) {
185 fInputFiles = new TObjArray();
186 TIter next(other.fInputFiles);
187 TObject *obj;
188 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
189 fInputFiles->SetOwner();
190 }
4e5c5506 191 if (other.fPackages) {
192 fPackages = new TObjArray();
193 TIter next(other.fPackages);
194 TObject *obj;
195 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
196 fPackages->SetOwner();
197 }
c57f56b7 198}
199
200//______________________________________________________________________________
201AliAnalysisAlien::~AliAnalysisAlien()
202{
203// Destructor.
204 if (fGridJDL) delete fGridJDL;
205 if (fInputFiles) delete fInputFiles;
4e5c5506 206 if (fPackages) delete fPackages;
c57f56b7 207}
208
209//______________________________________________________________________________
210AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
211{
212// Assignment.
213 if (this != &other) {
214 AliAnalysisGrid::operator=(other);
215 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
216 fPrice = other.fPrice;
217 fTTL = other.fTTL;
218 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
219 fMaxInitFailed = other.fMaxInitFailed;
220 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 221 fNtestFiles = other.fNtestFiles;
a3e84053 222 fNrunsPerMaster = other.fNrunsPerMaster;
223 fMaxMergeFiles = other.fMaxMergeFiles;
224 fNsubmitted = other.fNsubmitted;
225 fProductionMode = other.fProductionMode;
c57f56b7 226 fRunNumbers = other.fRunNumbers;
227 fExecutable = other.fExecutable;
0a1c1f7f 228 fExecutableCommand = other.fExecutableCommand;
c57f56b7 229 fArguments = other.fArguments;
230 fAnalysisMacro = other.fAnalysisMacro;
231 fAnalysisSource = other.fAnalysisSource;
232 fAdditionalLibs = other.fAdditionalLibs;
233 fSplitMode = other.fSplitMode;
234 fAPIVersion = other.fAPIVersion;
235 fROOTVersion = other.fROOTVersion;
236 fAliROOTVersion = other.fAliROOTVersion;
237 fUser = other.fUser;
238 fGridWorkingDir = other.fGridWorkingDir;
239 fGridDataDir = other.fGridDataDir;
240 fDataPattern = other.fDataPattern;
241 fGridOutputDir = other.fGridOutputDir;
242 fOutputArchive = other.fOutputArchive;
243 fOutputFiles = other.fOutputFiles;
244 fInputFormat = other.fInputFormat;
e7c71df0 245 fDatasetName = other.fDatasetName;
c57f56b7 246 fJDLName = other.fJDLName;
bb885a9e 247 fMergeExcludes = other.fMergeExcludes;
f965131e 248 fIncludePath = other.fIncludePath;
bb885a9e 249 fCloseSE = other.fCloseSE;
0df6ccf2 250 fFriendChainName = other.fFriendChainName;
c6cb3634 251 fJobTag = other.fJobTag;
c57f56b7 252 if (other.fInputFiles) {
253 fInputFiles = new TObjArray();
254 TIter next(other.fInputFiles);
255 TObject *obj;
256 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
257 fInputFiles->SetOwner();
258 }
4e5c5506 259 if (other.fPackages) {
260 fPackages = new TObjArray();
261 TIter next(other.fPackages);
262 TObject *obj;
263 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
264 fPackages->SetOwner();
265 }
c57f56b7 266 }
267 return *this;
268}
269
f965131e 270//______________________________________________________________________________
271void AliAnalysisAlien::AddIncludePath(const char *path)
272{
273// Add include path in the remote analysis macro.
274 TString p(path);
275 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
276 else fIncludePath += Form("-I%s ", path);
277}
278
c57f56b7 279//______________________________________________________________________________
280void AliAnalysisAlien::AddRunNumber(Int_t run)
281{
282// Add a run number to the list of runs to be processed.
283 if (fRunNumbers.Length()) fRunNumbers += " ";
284 fRunNumbers += Form("%d", run);
285}
286
ee75cfc3 287//______________________________________________________________________________
288void AliAnalysisAlien::AddRunNumber(const char* run)
289{
290// Add a run number to the list of runs to be processed.
291 if (fRunNumbers.Length()) fRunNumbers += " ";
292 fRunNumbers += run;
293}
294
c57f56b7 295//______________________________________________________________________________
296void AliAnalysisAlien::AddDataFile(const char *lfn)
297{
298// Adds a data file to the input to be analysed. The file should be a valid LFN
299// or point to an existing file in the alien workdir.
300 if (!fInputFiles) fInputFiles = new TObjArray();
301 fInputFiles->Add(new TObjString(lfn));
302}
303
304//______________________________________________________________________________
305Bool_t AliAnalysisAlien::Connect()
306{
307// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
308 if (gGrid && gGrid->IsConnected()) return kTRUE;
309 if (!gSystem->Getenv("alien_API_USER")) {
310 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
311 gSystem->Getenv("UID"));
312 return kFALSE;
313 }
314 if (!gGrid) {
315 Info("Connect", "Trying to connect to AliEn ...");
316 TGrid::Connect("alien://");
317 }
318 if (!gGrid || !gGrid->IsConnected()) {
319 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
320 return kFALSE;
321 }
322 fUser = gGrid->GetUser();
323 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
324 return kTRUE;
325}
326
327//______________________________________________________________________________
328void AliAnalysisAlien::CdWork()
329{
330// Check validity of alien workspace. Create directory if possible.
331 if (!Connect()) {
332 Error("CdWork", "Alien connection required");
333 return;
334 }
335 TString homedir = gGrid->GetHomeDirectory();
336 TString workdir = homedir + fGridWorkingDir;
923e2ca5 337 if (DirectoryExists(workdir)) {
338 gGrid->Cd(workdir);
339 return;
340 }
341 // Work directory not existing - create it
342 gGrid->Cd(homedir);
343 if (gGrid->Mkdir(workdir)) {
344 gGrid->Cd(fGridWorkingDir);
345 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
346 } else {
347 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
348 workdir.Data(), homedir.Data());
349 fGridWorkingDir = "";
350 }
c57f56b7 351}
352
353//______________________________________________________________________________
354Bool_t AliAnalysisAlien::CheckInputData()
355{
356// Check validity of input data. If necessary, create xml files.
a8739e8a 357 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 358 if (!fGridDataDir.Length()) {
359 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
360 return kFALSE;
361 }
362 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
363 return kTRUE;
c57f56b7 364 }
365 // Process declared files
366 Bool_t is_collection = kFALSE;
367 Bool_t is_xml = kFALSE;
368 Bool_t use_tags = kFALSE;
369 Bool_t checked = kFALSE;
370 CdWork();
371 TString file;
372 TString workdir = gGrid->GetHomeDirectory();
373 workdir += fGridWorkingDir;
374 if (fInputFiles) {
375 TObjString *objstr;
376 TIter next(fInputFiles);
377 while ((objstr=(TObjString*)next())) {
378 file = workdir;
379 file += "/";
380 file += objstr->GetString();
381 // Store full lfn path
382 if (FileExists(file)) objstr->SetString(file);
383 else {
384 file = objstr->GetName();
385 if (!FileExists(objstr->GetName())) {
386 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
387 objstr->GetName(), workdir.Data());
388 return kFALSE;
389 }
390 }
391 Bool_t iscoll, isxml, usetags;
392 CheckDataType(file, iscoll, isxml, usetags);
393 if (!checked) {
394 checked = kTRUE;
395 is_collection = iscoll;
396 is_xml = isxml;
397 use_tags = usetags;
398 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
399 } else {
400 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
401 Error("CheckInputData", "Some conflict was found in the types of inputs");
402 return kFALSE;
403 }
404 }
405 }
406 }
407 // Process requested run numbers
a8739e8a 408 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 409 // Check validity of alien data directory
410 if (!fGridDataDir.Length()) {
411 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
412 return kFALSE;
413 }
923e2ca5 414 if (!DirectoryExists(fGridDataDir)) {
c57f56b7 415 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
416 return kFALSE;
417 }
418 if (is_collection) {
419 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
420 return kFALSE;
421 }
422
423 if (checked && !is_xml) {
424 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
425 return kFALSE;
426 }
427 // Check validity of run number(s)
428 TObjArray *arr;
429 TObjString *os;
319593fb 430 Int_t nruns = 0;
431 TString schunk;
c57f56b7 432 TString path;
433 if (!checked) {
434 checked = kTRUE;
435 use_tags = fDataPattern.Contains("tag");
436 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
437 }
438 if (use_tags != fDataPattern.Contains("tag")) {
439 Error("CheckInputData", "Cannot mix input files using/not using tags");
440 return kFALSE;
441 }
442 if (fRunNumbers.Length()) {
a8739e8a 443 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 444 arr = fRunNumbers.Tokenize(" ");
445 TIter next(arr);
446 while ((os=(TObjString*)next())) {
447 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 448 if (!DirectoryExists(path)) {
449 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
a8739e8a 450 continue;
c57f56b7 451 }
452 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
453 TString msg = "\n##### file: ";
454 msg += path;
455 msg += " type: xml_collection;";
456 if (use_tags) msg += " using_tags: Yes";
457 else msg += " using_tags: No";
458 Info("CheckDataType", msg.Data());
319593fb 459 if (fNrunsPerMaster<2) {
d2a409b2 460 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 461 } else {
462 nruns++;
463 if (((nruns-1)%fNrunsPerMaster) == 0) {
464 schunk = os->GetString();
465 }
466 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
467 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 468 AddDataFile(schunk);
319593fb 469 }
c57f56b7 470 }
471 delete arr;
a8739e8a 472 } else {
473 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
474 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
475 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 476 if (!DirectoryExists(path)) {
477// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
a8739e8a 478 continue;
479 }
480 path = Form("%s/%d.xml", workdir.Data(),irun);
481 TString msg = "\n##### file: ";
482 msg += path;
483 msg += " type: xml_collection;";
484 if (use_tags) msg += " using_tags: Yes";
485 else msg += " using_tags: No";
486 Info("CheckDataType", msg.Data());
319593fb 487 if (fNrunsPerMaster<2) {
d2a409b2 488 AddDataFile(Form("%d.xml",irun));
319593fb 489 } else {
490 nruns++;
491 if (((nruns-1)%fNrunsPerMaster) == 0) {
492 schunk = Form("%d", irun);
493 }
494 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
495 schunk += Form("_%d.xml", irun);
d2a409b2 496 AddDataFile(schunk);
319593fb 497 }
a8739e8a 498 }
c57f56b7 499 }
500 return kTRUE;
501}
502
503//______________________________________________________________________________
504Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
505{
506// Create dataset for the grid data directory + run number.
507 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
508 if (!Connect()) {
509 Error("CreateDataset", "Cannot create dataset with no grid connection");
510 return kFALSE;
511 }
512
513 // Cd workspace
514 CdWork();
515 TString workdir = gGrid->GetHomeDirectory();
516 workdir += fGridWorkingDir;
517
518 // Compose the 'find' command arguments
519 TString command;
520 TString options = "-x collection ";
bb885a9e 521 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 522 TString conditions = "";
523
524 TString file;
525 TString path;
319593fb 526 Int_t nruns = 0;
527 TString schunk;
ab254fd1 528 TGridCollection *cbase=0, *cadd=0;
d2a409b2 529 if (!fRunNumbers.Length() && !fRunRange[0]) {
530 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
531 // Make a single data collection from data directory.
532 path = fGridDataDir;
923e2ca5 533 if (!DirectoryExists(path)) {
d2a409b2 534 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
535 return kFALSE;
536 }
923e2ca5 537// CdWork();
d2a409b2 538 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
539 else file = Form("%s.xml", gSystem->BaseName(path));
540 if (gSystem->AccessPathName(file)) {
541 command = "find ";
542 command += options;
543 command += path;
544 command += " ";
545 command += pattern;
546 command += conditions;
923e2ca5 547 Printf("command: %s", command.Data());
d2a409b2 548 TGridResult *res = gGrid->Command(command);
549 if (res) delete res;
550 // Write standard output to file
551 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
552 }
553 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
554 // Copy xml file to alien space
555 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
556 if (!FileExists(file)) {
557 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
558 return kFALSE;
559 }
560 // Update list of files to be processed.
561 }
562 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
563 return kTRUE;
564 }
c57f56b7 565 // Several runs
a8739e8a 566 if (fRunNumbers.Length()) {
567 TObjArray *arr = fRunNumbers.Tokenize(" ");
568 TObjString *os;
569 TIter next(arr);
570 while ((os=(TObjString*)next())) {
571 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
923e2ca5 572 if (!DirectoryExists(path)) continue;
573// CdWork();
a8739e8a 574 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
575 else file = Form("%s.xml", os->GetString().Data());
319593fb 576 // If local collection file does not exist, create it via 'find' command.
577 if (gSystem->AccessPathName(file)) {
578 command = "find ";
579 command += options;
580 command += path;
581 command += pattern;
582 command += conditions;
583 TGridResult *res = gGrid->Command(command);
584 if (res) delete res;
585 // Write standard output to file
586 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
587 }
a8739e8a 588 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 589 // Check if there is one run per master job.
590 if (fNrunsPerMaster<2) {
591 if (FileExists(file)) {
592 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
593 continue;
594 }
595 // Copy xml file to alien space
596 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
597 if (!FileExists(file)) {
598 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
599 delete arr;
600 return kFALSE;
601 }
602 } else {
603 nruns++;
604 if (((nruns-1)%fNrunsPerMaster) == 0) {
605 schunk = os->GetString();
606 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
607 } else {
608 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
923e2ca5 609 Printf(" Merging collection <%s> into masterjob input...", file.Data());
319593fb 610 cbase->Add(cadd);
611 delete cadd;
612 }
613 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
614 continue;
615 }
616 schunk += Form("_%s.xml", os->GetString().Data());
617 if (FileExists(schunk)) {
618 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
619 continue;
620 }
923e2ca5 621 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 622 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
623 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
624 if (!FileExists(schunk)) {
625 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
626 delete arr;
627 return kFALSE;
628 }
629 }
a8739e8a 630 }
631 delete arr;
632 } else {
633 // Process a full run range.
634 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
635 path = Form("%s/%d ", fGridDataDir.Data(), irun);
923e2ca5 636 if (!DirectoryExists(path)) continue;
637// CdWork();
a8739e8a 638 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
639 else file = Form("%d.xml", irun);
319593fb 640 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 641 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
642// gGrid->Rm(file);
643 continue;
644 }
319593fb 645 // If local collection file does not exist, create it via 'find' command.
646 if (gSystem->AccessPathName(file)) {
647 command = "find ";
648 command += options;
649 command += path;
650 command += pattern;
651 command += conditions;
652 TGridResult *res = gGrid->Command(command);
653 if (res) delete res;
654 // Write standard output to file
655 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
656 }
a8739e8a 657 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 658 // Check if there is one run per master job.
659 if (fNrunsPerMaster<2) {
660 if (FileExists(file)) {
661 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
662 continue;
663 }
664 // Copy xml file to alien space
665 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
666 if (!FileExists(file)) {
667 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
668 return kFALSE;
669 }
670 } else {
671 nruns++;
95e5b448 672 // Check if the collection for the chunk exist locally.
673 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
674 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
923e2ca5 675 Printf(" Merging collection <%s> into %d runs chunk...",file.Data(),fNrunsPerMaster);
319593fb 676 if (((nruns-1)%fNrunsPerMaster) == 0) {
677 schunk = Form("%d", irun);
678 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
679 } else {
680 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
681 cbase->Add(cadd);
682 delete cadd;
683 }
684 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
685 continue;
686 }
687 schunk += Form("_%d.xml", irun);
688 if (FileExists(schunk)) {
689 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
690 continue;
691 }
923e2ca5 692 Printf("Exporting merged collection <%s> and copying to AliEn.", schunk.Data());
319593fb 693 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 694 if (FileExists(schunk)) {
695 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
696 continue;
697 }
319593fb 698 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
699 if (!FileExists(schunk)) {
700 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
701 return kFALSE;
702 }
703 }
c57f56b7 704 }
a8739e8a 705 }
c57f56b7 706 return kTRUE;
707}
708
709//______________________________________________________________________________
710Bool_t AliAnalysisAlien::CreateJDL()
711{
712// Generate a JDL file according to current settings. The name of the file is
713// specified by fJDLName.
714 Bool_t error = kFALSE;
715 TObjArray *arr = 0;
716 Bool_t copy = kTRUE;
717 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
718 Bool_t generate = kTRUE;
719 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
720 if (!Connect()) {
721 Error("CreateJDL", "Alien connection required");
722 return kFALSE;
723 }
724 // Check validity of alien workspace
725 CdWork();
726 TString workdir = gGrid->GetHomeDirectory();
727 workdir += fGridWorkingDir;
728 if (generate) {
729 TObjString *os;
730 if (!fInputFiles) {
731 Error("CreateJDL()", "Define some input files for your analysis.");
732 error = kTRUE;
733 }
734 // Compose list of input files
735 // Check if output files were defined
736 if (!fOutputFiles.Length()) {
737 Error("CreateJDL", "You must define at least one output file");
738 error = kTRUE;
739 }
740 // Check if an output directory was defined and valid
741 if (!fGridOutputDir.Length()) {
742 Error("CreateJDL", "You must define AliEn output directory");
743 error = kTRUE;
744 } else {
d2a409b2 745 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
923e2ca5 746 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 747 if (gGrid->Mkdir(fGridOutputDir)) {
748 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
749 } else {
750 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 751 // error = kTRUE;
c57f56b7 752 }
753 }
754 gGrid->Cd(workdir);
755 }
756 // Exit if any error up to now
757 if (error) return kFALSE;
758 // Set JDL fields
759 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 760 fGridJDL->SetExecutable(fExecutable);
c57f56b7 761// fGridJDL->SetTTL((UInt_t)fTTL);
762 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
763 if (fMaxInitFailed > 0)
764 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
765 if (fSplitMaxInputFileNumber > 0)
766 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
767 if (fSplitMode.Length())
768 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
769// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
770 if (fAliROOTVersion.Length())
771 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
772 if (fROOTVersion.Length())
773 fGridJDL->AddToPackages("ROOT", fROOTVersion);
774 if (fAPIVersion.Length())
775 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
776 fGridJDL->SetInputDataListFormat(fInputFormat);
777 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 778 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
f10e8481 779 TString analysisFile = fExecutable;
780 analysisFile.ReplaceAll(".sh", ".root");
781 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 782 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
783 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
784 if (fAdditionalLibs.Length()) {
785 arr = fAdditionalLibs.Tokenize(" ");
786 TIter next(arr);
787 while ((os=(TObjString*)next())) {
788 if (os->GetString().Contains(".so")) continue;
789 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
790 }
791 delete arr;
792 }
4e5c5506 793 if (fPackages) {
794 TIter next(fPackages);
795 TObject *obj;
796 while ((obj=next()))
797 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
798 }
c57f56b7 799 if (fOutputArchive.Length()) {
800 arr = fOutputArchive.Tokenize(" ");
801 TIter next(arr);
802 while ((os=(TObjString*)next()))
43da816a 803 if (!os->GetString().Contains("@") && fCloseSE.Length())
804 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
805 else
806 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 807 delete arr;
808 }
c57f56b7 809 arr = fOutputFiles.Tokenize(" ");
810 TIter next(arr);
43da816a 811 while ((os=(TObjString*)next())) {
e1eaf596 812 // Ignore ouputs in jdl that are also in outputarchive
813 TString sout = os->GetString();
814 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
815 if (fOutputArchive.Contains(sout)) continue;
43da816a 816 if (!os->GetString().Contains("@") && fCloseSE.Length())
817 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
818 else
819 fGridJDL->AddToOutputSandbox(os->GetString());
820 }
c57f56b7 821 delete arr;
822// fGridJDL->SetPrice((UInt_t)fPrice);
823 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
0d5d317c 824 TString validationScript = fExecutable;
825 validationScript.ReplaceAll(".sh", "_validation.sh");
826 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 827 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 828 // Write a jdl with 2 input parameters: collection name and output dir name.
829 WriteJDL(copy);
c57f56b7 830 }
831 // Copy jdl to grid workspace
a8739e8a 832 if (copy) {
c57f56b7 833 if (fAdditionalLibs.Length()) {
e7c71df0 834 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 835 TObjString *os;
836 TIter next(arr);
837 while ((os=(TObjString*)next())) {
c57f56b7 838 if (os->GetString().Contains(".so")) continue;
4e5c5506 839 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 840 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
841 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
842 }
843 delete arr;
844 }
4e5c5506 845 if (fPackages) {
846 TIter next(fPackages);
847 TObject *obj;
848 while ((obj=next())) {
849 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
850 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
851 }
852 }
c57f56b7 853 }
854 return kTRUE;
855}
856
a8739e8a 857//______________________________________________________________________________
d2a409b2 858Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 859{
860// Writes one or more JDL's corresponding to findex. If findex is negative,
861// all run numbers are considered in one go (jdl). For non-negative indices
862// they correspond to the indices in the array fInputFiles.
863 if (!fInputFiles) return kFALSE;
864 TObjString *os;
a8739e8a 865 TString workdir = gGrid->GetHomeDirectory();
866 workdir += fGridWorkingDir;
d2a409b2 867
868 if (!fRunNumbers.Length() && !fRunRange[0]) {
869 // One jdl with no parameters in case input data is specified by name.
a8739e8a 870 TIter next(fInputFiles);
871 while ((os=(TObjString*)next()))
872 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 873 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 874 } else {
d2a409b2 875 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
876 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
e1eaf596 877 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 878 }
879
880
881 // Generate the JDL as a string
882 TString sjdl = fGridJDL->Generate();
883 Int_t index;
884 index = sjdl.Index("Executable");
885 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
886 index = sjdl.Index("Split ");
887 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
888 index = sjdl.Index("SplitMaxInputFileNumber");
889 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
890 index = sjdl.Index("InputDataCollection");
891 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 892 index = sjdl.Index("InputFile");
893 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
894 index = sjdl.Index("InputDataList ");
895 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
896 index = sjdl.Index("InputDataListFormat");
897 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
898 index = sjdl.Index("Price");
899 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
900 index = sjdl.Index("Requirements");
901 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
902 index = sjdl.Index("Packages");
903 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
904 index = sjdl.Index("User =");
905 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
906 index = sjdl.Index("TTL");
907 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
908 index = sjdl.Index("OutputFile");
909 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
910 index = sjdl.Index("OutputDir");
911 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
912 index = sjdl.Index("OutputArchive");
913 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
914 index = sjdl.Index("MaxInitFailed");
915 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
916 index = sjdl.Index("MasterResubmitThreshold");
917 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
918 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
919 index = sjdl.Index("Validationcommand");
920 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
921 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
922 sjdl.ReplaceAll("(member", "\n (member");
923 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
924 sjdl.ReplaceAll("{", "{\n ");
925 sjdl.ReplaceAll("};", "\n};");
926 sjdl.ReplaceAll("{\n \n", "{\n");
927 sjdl.ReplaceAll("\n\n", "\n");
928 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
929 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
f60ef0ba 930 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
a8739e8a 931 index = sjdl.Index("JDLVariables");
932 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
933 // Write jdl to file
a8739e8a 934 ofstream out;
d2a409b2 935 out.open(fJDLName.Data(), ios::out);
a8739e8a 936 if (out.bad()) {
d2a409b2 937 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 938 return kFALSE;
939 }
940 out << sjdl << endl;
941
942 // Copy jdl to grid workspace
943 if (!copy) {
944 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
945 } else {
d2a409b2 946 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
947 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 948 if (fProductionMode)
b5fe9cba 949 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 950 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 951 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 952 }
953 return kTRUE;
954}
955
c57f56b7 956//______________________________________________________________________________
5513444a 957Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 958{
959// Returns true if file exists.
5513444a 960 if (!gGrid) return kFALSE;
c57f56b7 961 TGridResult *res = gGrid->Ls(lfn);
962 if (!res) return kFALSE;
963 TMap *map = dynamic_cast<TMap*>(res->At(0));
964 if (!map) {
965 delete res;
966 return kFALSE;
967 }
968 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
969 if (!objs || !objs->GetString().Length()) {
970 delete res;
971 return kFALSE;
972 }
973 delete res;
974 return kTRUE;
975}
976
923e2ca5 977//______________________________________________________________________________
978Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
979{
980// Returns true if directory exists. Can be also a path.
981 if (!gGrid) return kFALSE;
982 // Check if dirname is a path
983 TString dirstripped = dirname;
984 dirstripped = dirstripped.Strip();
985 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
986 TString dir = gSystem->BaseName(dirstripped);
987 dir += "/";
988 TString path = gSystem->DirName(dirstripped);
989 TGridResult *res = gGrid->Ls(path, "-F");
990 if (!res) return kFALSE;
991 TIter next(res);
992 TMap *map;
993 TObject *obj;
994 while ((map=dynamic_cast<TMap*>(next()))) {
995 obj = map->GetValue("name");
996 if (!obj) break;
997 if (dir == obj->GetName()) {
998 delete res;
999 return kTRUE;
1000 }
1001 }
1002 delete res;
1003 return kFALSE;
1004}
1005
c57f56b7 1006//______________________________________________________________________________
1007void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1008{
1009// Check input data type.
1010 is_collection = kFALSE;
1011 is_xml = kFALSE;
1012 use_tags = kFALSE;
1013 if (!gGrid) {
1014 Error("CheckDataType", "No connection to grid");
1015 return;
1016 }
1017 is_collection = IsCollection(lfn);
1018 TString msg = "\n##### file: ";
1019 msg += lfn;
1020 if (is_collection) {
1021 msg += " type: raw_collection;";
1022 // special treatment for collections
1023 is_xml = kFALSE;
1024 // check for tag files in the collection
1025 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1026 if (!res) {
1027 msg += " using_tags: No (unknown)";
1028 Info("CheckDataType", msg.Data());
1029 return;
1030 }
1031 const char* typeStr = res->GetKey(0, "origLFN");
1032 if (!typeStr || !strlen(typeStr)) {
1033 msg += " using_tags: No (unknown)";
1034 Info("CheckDataType", msg.Data());
1035 return;
1036 }
1037 TString file = typeStr;
1038 use_tags = file.Contains(".tag");
1039 if (use_tags) msg += " using_tags: Yes";
1040 else msg += " using_tags: No";
1041 Info("CheckDataType", msg.Data());
1042 return;
1043 }
1044 TString slfn(lfn);
1045 slfn.ToLower();
1046 is_xml = slfn.Contains(".xml");
1047 if (is_xml) {
1048 // Open xml collection and check if there are tag files inside
1049 msg += " type: xml_collection;";
1050 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1051 if (!coll) {
1052 msg += " using_tags: No (unknown)";
1053 Info("CheckDataType", msg.Data());
1054 return;
1055 }
1056 TMap *map = coll->Next();
1057 if (!map) {
1058 msg += " using_tags: No (unknown)";
1059 Info("CheckDataType", msg.Data());
1060 return;
1061 }
1062 map = (TMap*)map->GetValue("");
1063 TString file;
1064 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1065 use_tags = file.Contains(".tag");
1066 delete coll;
1067 if (use_tags) msg += " using_tags: Yes";
1068 else msg += " using_tags: No";
1069 Info("CheckDataType", msg.Data());
1070 return;
1071 }
1072 use_tags = slfn.Contains(".tag");
1073 if (slfn.Contains(".root")) msg += " type: root file;";
1074 else msg += " type: unhnown file;";
1075 if (use_tags) msg += " using_tags: Yes";
1076 else msg += " using_tags: No";
1077 Info("CheckDataType", msg.Data());
1078}
1079
4e5c5506 1080//______________________________________________________________________________
1081void AliAnalysisAlien::EnablePackage(const char *package)
1082{
1083// Enables a par file supposed to exist in the current directory.
1084 TString pkg(package);
1085 pkg.ReplaceAll(".par", "");
1086 pkg += ".par";
1087 if (gSystem->AccessPathName(pkg)) {
1088 Error("EnablePackage", "Package %s not found", pkg.Data());
1089 return;
1090 }
1091 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1092 Info("EnablePackage", "AliEn plugin will use .par packages");
1093 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1094 if (!fPackages) {
1095 fPackages = new TObjArray();
1096 fPackages->SetOwner();
1097 }
1098 fPackages->Add(new TObjString(pkg));
1099}
1100
d2a409b2 1101//______________________________________________________________________________
1102const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1103{
1104// Get job status for all jobs with jobid>jobidstart.
1105 static char mstatus[20];
1106 mstatus[0] = '\0';
1107 nrunning = 0;
1108 nwaiting = 0;
1109 nerror = 0;
1110 ndone = 0;
1111 TGridJobStatusList *list = gGrid->Ps("");
1112 if (!list) return mstatus;
1113 Int_t nentries = list->GetSize();
1114 TGridJobStatus *status;
1115 Int_t pid;
1116 for (Int_t ijob=0; ijob<nentries; ijob++) {
1117 status = (TGridJobStatus *)list->At(ijob);
1118 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1119 if (pid<jobidstart) continue;
1120 if (pid == lastid) {
1121 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1122 }
1123 switch (status->GetStatus()) {
1124 case TGridJobStatus::kWAITING:
1125 nwaiting++; break;
1126 case TGridJobStatus::kRUNNING:
1127 nrunning++; break;
1128 case TGridJobStatus::kABORTED:
1129 case TGridJobStatus::kFAIL:
1130 case TGridJobStatus::kUNKNOWN:
1131 nerror++; break;
1132 case TGridJobStatus::kDONE:
1133 ndone++;
1134 }
1135 }
1136 list->Delete();
1137 delete list;
1138 return mstatus;
1139}
1140
c57f56b7 1141//______________________________________________________________________________
1142Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1143{
1144// Returns true if file is a collection. Functionality duplicated from
1145// TAlien::Type() because we don't want to directly depend on TAlien.
1146 if (!gGrid) {
1147 Error("IsCollection", "No connection to grid");
1148 return kFALSE;
1149 }
1150 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1151 if (!res) return kFALSE;
1152 const char* typeStr = res->GetKey(0, "type");
1153 if (!typeStr || !strlen(typeStr)) return kFALSE;
1154 if (!strcmp(typeStr, "collection")) return kTRUE;
1155 delete res;
1156 return kFALSE;
1157}
1158
16a4353c 1159//______________________________________________________________________________
1160void AliAnalysisAlien::Print(Option_t *) const
1161{
1162// Print current plugin settings.
923e2ca5 1163 Printf("### AliEn analysis plugin current settings ###");
1164 Printf("= Production mode:______________________________ %d", fProductionMode);
1165 Printf("= Version of API requested: ____________________ %s", fAPIVersion.Data());
1166 Printf("= Version of ROOT requested: ___________________ %s", fROOTVersion.Data());
1167 Printf("= Version of AliRoot requested: ________________ %s", fAliROOTVersion.Data());
16a4353c 1168 if (fUser.Length())
923e2ca5 1169 Printf("= User running the plugin: _____________________ %s", fUser.Data());
1170 Printf("= Grid workdir relative to user $HOME: _________ %s", fGridWorkingDir.Data());
1171 Printf("= Grid output directory relative to workdir: ___ %s", fGridOutputDir.Data());
1172 Printf("= Data base directory path requested: __________ %s", fGridDataDir.Data());
1173 Printf("= Data search pattern: _________________________ %s", fDataPattern.Data());
1174 Printf("= Input data format: ___________________________ %s", fInputFormat.Data());
16a4353c 1175 if (fRunNumbers.Length())
923e2ca5 1176 Printf("= Run numbers to be processed: _________________ %s", fRunNumbers.Data());
16a4353c 1177 if (fRunRange[0])
923e2ca5 1178 Printf("= Run range to be processed: ___________________ %d-%d", fRunRange[0], fRunRange[1]);
16a4353c 1179 if (!fRunRange[0] && !fRunNumbers.Length()) {
1180 TIter next(fInputFiles);
1181 TObject *obj;
1182 TString list;
1183 while ((obj=next())) list += obj->GetName();
923e2ca5 1184 Printf("= Input files to be processed: _________________ %s", list.Data());
16a4353c 1185 }
1186 if (TestBit(AliAnalysisGrid::kTest))
923e2ca5 1187 Printf("= Number of input files used in test mode: _____ %d", fNtestFiles);
1188 Printf("= List of output files to be registered: _______ %s", fOutputFiles.Data());
1189 Printf("= List of outputs going to be archived: ________ %s", fOutputArchive.Data());
1190 Printf("= List of outputs that should not be merged: ___ %s", fMergeExcludes.Data());
1191 Printf("=====================================================================");
1192 Printf("= Job price: ___________________________________ %d", fPrice);
1193 Printf("= Time to live (TTL): __________________________ %d", fTTL);
1194 Printf("= Max files per subjob: ________________________ %d", fSplitMaxInputFileNumber);
16a4353c 1195 if (fMaxInitFailed>0)
923e2ca5 1196 Printf("= Max number of subjob fails to kill: __________ %d", fMaxInitFailed);
16a4353c 1197 if (fMasterResubmitThreshold>0)
923e2ca5 1198 Printf("= Resubmit master job if failed subjobs >_______ %d", fMasterResubmitThreshold);
319593fb 1199 if (fNrunsPerMaster>0)
923e2ca5 1200 Printf("= Number of runs per master job: _______________ %d", fNrunsPerMaster);
1201 Printf("= Number of files in one chunk to be merged: ___ %d", fMaxMergeFiles);
1202 Printf("= Name of the generated execution script: ______ %s",fExecutable.Data());
16a4353c 1203 if (fArguments.Length())
923e2ca5 1204 Printf("= Arguments for the execution script: __________ %s",fArguments.Data());
1205 Printf("= Name of the generated analysis macro: ________ %s",fAnalysisMacro.Data());
1206 Printf("= User analysis files to be deployed: __________ %s",fAnalysisSource.Data());
1207 Printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>",fAdditionalLibs.Data());
1208 Printf("= Master jobs split mode: ______________________ %s",fSplitMode.Data());
16a4353c 1209 if (fDatasetName)
923e2ca5 1210 Printf("= Custom name for the dataset to be created: ___ %s", fDatasetName.Data());
1211 Printf("= Name of the generated JDL: ___________________ %s", fJDLName.Data());
16a4353c 1212 if (fIncludePath.Data())
923e2ca5 1213 Printf("= Include path for runtime task compilation: ___ %s", fIncludePath.Data());
16a4353c 1214 if (fCloseSE.Length())
923e2ca5 1215 Printf("= Force job outputs to storage element: ________ %s", fCloseSE.Data());
16a4353c 1216 if (fFriendChainName.Length())
923e2ca5 1217 Printf("= Open friend chain file on worker: ____________ %s", fFriendChainName.Data());
16a4353c 1218 if (fPackages) {
1219 TIter next(fPackages);
1220 TObject *obj;
1221 TString list;
1222 while ((obj=next())) list += obj->GetName();
923e2ca5 1223 Printf("= Par files to be used: ________________________ %s", list.Data());
16a4353c 1224 }
1225}
1226
c57f56b7 1227//______________________________________________________________________________
1228void AliAnalysisAlien::SetDefaults()
1229{
1230// Set default values for everything. What cannot be filled will be left empty.
1231 if (fGridJDL) delete fGridJDL;
1232 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1233 fPrice = 1;
1234 fTTL = 30000;
1235 fSplitMaxInputFileNumber = 100;
1236 fMaxInitFailed = 0;
1237 fMasterResubmitThreshold = 0;
bb885a9e 1238 fNtestFiles = 10;
a8739e8a 1239 fRunRange[0] = 0;
1240 fRunRange[1] = 0;
319593fb 1241 fNrunsPerMaster = 1;
16a4353c 1242 fMaxMergeFiles = 100;
c57f56b7 1243 fRunNumbers = "";
1244 fExecutable = "analysis.sh";
0a1c1f7f 1245 fExecutableCommand = "root -b -q";
c57f56b7 1246 fArguments = "";
1247 fAnalysisMacro = "myAnalysis.C";
1248 fAnalysisSource = "";
1249 fAdditionalLibs = "";
1250 fSplitMode = "se";
1251 fAPIVersion = "";
1252 fROOTVersion = "";
1253 fAliROOTVersion = "";
1254 fUser = ""; // Your alien user name
1255 fGridWorkingDir = "";
1256 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1257 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1258 fFriendChainName = "";
c57f56b7 1259 fGridOutputDir = "output";
1260 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1261 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1262 fInputFormat = "xml-single";
1263 fJDLName = "analysis.jdl";
c6cb3634 1264 fJobTag = "Automatically generated analysis JDL";
bb885a9e 1265 fMergeExcludes = "";
c57f56b7 1266}
1267
1268//______________________________________________________________________________
1269Bool_t AliAnalysisAlien::MergeOutputs()
1270{
1271// Merge analysis outputs existing in the AliEn space.
1272 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1273 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1274 if (!Connect()) {
1275 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1276 return kFALSE;
1277 }
1278 // Get the output path
d2a409b2 1279 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
923e2ca5 1280 if (!DirectoryExists(fGridOutputDir)) {
c57f56b7 1281 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1282 return kFALSE;
1283 }
1284 if (!fOutputFiles.Length()) {
1285 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1286 return kFALSE;
1287 }
1288 TObjArray *list = fOutputFiles.Tokenize(" ");
1289 TIter next(list);
1290 TObjString *str;
1291 TString command;
1292 TString output_file;
16a4353c 1293 TString output_chunk;
1294 TString previous_chunk;
1295 Int_t count_chunk = 0;
1296 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1297 Bool_t merged = kTRUE;
1298 while((str=(TObjString*)next())) {
1299 output_file = str->GetString();
1300 Int_t index = output_file.Index("@");
1301 if (index > 0) output_file.Remove(index);
319593fb 1302 // Skip already merged outputs
1303 if (!gSystem->AccessPathName(output_file)) {
1304 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1305 continue;
1306 }
bb885a9e 1307 if (fMergeExcludes.Length() &&
1308 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1309 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1310 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
923e2ca5 1311 Printf("command: %s", command.Data());
c57f56b7 1312 TGridResult *res = gGrid->Command(command);
1313 if (!res) continue;
1314 TFileMerger *fm = 0;
1315 TIter nextmap(res);
ab254fd1 1316 TMap *map = 0;
16a4353c 1317 previous_chunk = "";
1318 count_chunk = 0;
319593fb 1319 // Check if there is a merge operation to resume
1320 output_chunk = output_file;
1321 output_chunk.ReplaceAll(".root", "_*.root");
1322 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1323 while (1) {
1324 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1325 if (!map) {
1326 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1327 delete res;
1328 return kFALSE;
1329 }
1330 output_chunk = output_file;
1331 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
923e2ca5 1332 Printf("%s", output_chunk.Data());
319593fb 1333 count_chunk++;
1334 if (gSystem->AccessPathName(output_chunk)) continue;
1335 // Merged file with chunks up to <count_chunk> found
923e2ca5 1336 Printf("Resume merging of <%s> from <%s>", output_file.Data(), output_chunk.Data());
319593fb 1337 previous_chunk = output_chunk;
1338 break;
1339 }
1340 }
16a4353c 1341 count_zero = fMaxMergeFiles;
c57f56b7 1342 while ((map=(TMap*)nextmap())) {
16a4353c 1343 // Loop 'find' results and get next LFN
1344 if (count_zero == fMaxMergeFiles) {
1345 // First file in chunk - create file merger and add previous chunk if any.
1346 fm = new TFileMerger(kFALSE);
1347 fm->SetFastMethod(kTRUE);
1348 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1349 output_chunk = output_file;
1350 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1351 }
1352 // If last file found, put merged results in the output file
1353 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1354 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1355 if (!objs || !objs->GetString().Length()) {
16a4353c 1356 // Nothing found - skip this output
c57f56b7 1357 delete res;
16a4353c 1358 delete fm;
1359 break;
1360 }
1361 // Add file to be merged and decrement chunk counter.
1362 fm->AddFile(objs->GetString());
1363 count_zero--;
1364 if (count_zero==0 || map == res->Last()) {
1365 fm->OutputFile(output_chunk);
1366 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1367 // Nothing found - skip this output
1368 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1369 delete res;
1370 delete fm;
1371 break;
1372 }
1373 // Merge the outputs, then go to next chunk
1374 if (!fm->Merge()) {
1375 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1376 delete res;
1377 delete fm;
1378 merged = kFALSE;
1379 break;
1380 } else {
1381 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1382 gSystem->Unlink(previous_chunk);
16a4353c 1383 }
1384 if (map == res->Last()) {
1385 delete res;
1386 delete fm;
1387 break;
1388 }
1389 count_chunk++;
1390 count_zero = fMaxMergeFiles;
1391 previous_chunk = output_chunk;
c57f56b7 1392 }
c57f56b7 1393 }
c57f56b7 1394 }
1395 if (!merged) {
1396 Error("MergeOutputs", "Terminate() will NOT be executed");
1397 }
1398 return merged;
1399}
1400
bb885a9e 1401//______________________________________________________________________________
1402void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1403{
1404// Use the output files connected to output containers from the analysis manager
1405// rather than the files defined by SetOutputFiles
1406 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1407 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1408 analysis manager");
1409 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1410}
1411
5513444a 1412//______________________________________________________________________________
1413Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1414{
1415// Start remote grid analysis.
1416
43da816a 1417 // Check if output files have to be taken from the analysis manager
1418 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1419 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1420 if (!mgr || !mgr->IsInitialized()) {
1421 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1422 return kFALSE;
43da816a 1423 }
1424 fOutputFiles = "";
1425 TIter next(mgr->GetOutputs());
1426 AliAnalysisDataContainer *output;
1427 while ((output=(AliAnalysisDataContainer*)next())) {
1428 const char *filename = output->GetFileName();
1429 if (!(strcmp(filename, "default"))) {
1430 if (!mgr->GetOutputEventHandler()) continue;
1431 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1432 }
1433 if (fOutputFiles.Length()) fOutputFiles += " ";
1434 fOutputFiles += filename;
1435 }
c07b9ce2 1436 // Add extra files registered to the analysis manager
1437 if (mgr->GetExtraFiles().Length()) {
1438 if (fOutputFiles.Length()) fOutputFiles += " ";
1439 fOutputFiles += mgr->GetExtraFiles();
1440 }
43da816a 1441 }
f7b1cbc2 1442// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1443 if (TestBit(AliAnalysisGrid::kOffline)) {
1444 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1445 \n there nor any job run. You can revise the JDL and analysis \
1446 \n macro then run the same in \"submit\" mode.");
1447 } else if (TestBit(AliAnalysisGrid::kTest)) {
1448 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1449 \n dataset.");
1450 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1451 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1452 \n space and job submitted.");
1453 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1454 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1455 return kTRUE;
c57f56b7 1456 } else {
1457 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1458 }
1459
1460 if (!Connect()) {
1461 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1462 return kFALSE;
16a4353c 1463 }
1464 Print();
c57f56b7 1465 if (!CheckInputData()) {
1466 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1467 return kFALSE;
c57f56b7 1468 }
1469 CreateDataset(fDataPattern);
1470 WriteAnalysisFile();
1471 WriteAnalysisMacro();
1472 WriteExecutable();
1473 WriteValidationScript();
5513444a 1474 if (!CreateJDL()) return kFALSE;
1475 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1476 if (TestBit(AliAnalysisGrid::kTest)) {
1477 // Locally testing the analysis
1478 Info("StartAnalysis", "\n_______________________________________________________________________ \
1479 \n Running analysis script in a daughter shell as on a worker node \
1480 \n_______________________________________________________________________");
1481 TObjArray *list = fOutputFiles.Tokenize(" ");
1482 TIter next(list);
1483 TObjString *str;
1484 TString output_file;
1485 while((str=(TObjString*)next())) {
1486 output_file = str->GetString();
1487 Int_t index = output_file.Index("@");
1488 if (index > 0) output_file.Remove(index);
43da816a 1489 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1490 }
1491 delete list;
1492 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
0d5d317c 1493 TString validationScript = fExecutable;
1494 validationScript.ReplaceAll(".sh", "_validation.sh");
1495 gSystem->Exec(Form("bash %s",validationScript.Data()));
c57f56b7 1496// gSystem->Exec("cat stdout");
5513444a 1497 return kFALSE;
c57f56b7 1498 }
5513444a 1499 // Check if submitting is managed by LPM manager
a3e84053 1500 if (fProductionMode) {
5513444a 1501 TString prodfile = fJDLName;
1502 prodfile.ReplaceAll(".jdl", ".prod");
1503 WriteProductionFile(prodfile);
1504 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1505 return kFALSE;
1506 }
a8739e8a 1507 // Submit AliEn job(s)
d2a409b2 1508 gGrid->Cd(fGridOutputDir);
a8739e8a 1509 TGridResult *res;
c57f56b7 1510 TString jobID = "";
d2a409b2 1511 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1512 // Submit a given xml or a set of runs
a8739e8a 1513 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
923e2ca5 1514 Printf("*************************** %s",Form("submit %s", fJDLName.Data()));
a8739e8a 1515 if (res) {
1516 const char *cjobId = res->GetKey(0,"jobId");
1517 if (!cjobId) {
1518 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1519 return kFALSE;
a8739e8a 1520 } else {
1521 Info("StartAnalysis", "\n_______________________________________________________________________ \
1522 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1523 \n_______________________________________________________________________",
1524 fJDLName.Data(), cjobId);
1525 jobID = cjobId;
1526 }
1527 delete res;
1528 }
1529 } else {
d2a409b2 1530 // Submit for a range of enumeration of runs.
1531 Submit();
c57f56b7 1532 }
a8739e8a 1533
c57f56b7 1534 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1535 \n You may exit at any time and terminate the job later using the option <terminate> \
1536 \n ##################################################################################", jobID.Data());
bb885a9e 1537 gSystem->Exec("aliensh");
5513444a 1538 return kTRUE;
c57f56b7 1539}
1540
d2a409b2 1541//______________________________________________________________________________
1542void AliAnalysisAlien::Submit()
1543{
1544// Submit all master jobs.
1545 Int_t nmasterjobs = fInputFiles->GetEntries();
1546 Long_t tshoot = gSystem->Now();
1547 if (!fNsubmitted) SubmitNext();
1548 while (fNsubmitted < nmasterjobs) {
1549 Long_t now = gSystem->Now();
1550 if ((now-tshoot)>30000) {
1551 tshoot = now;
1552 SubmitNext();
1553 }
1554 }
1555}
1556
1557//______________________________________________________________________________
1558void AliAnalysisAlien::SubmitNext()
1559{
1560// Submit next bunch of master jobs if the queue is free.
1561 static Bool_t iscalled = kFALSE;
1562 static Int_t firstmaster = 0;
1563 static Int_t lastmaster = 0;
1564 static Int_t npermaster = 0;
1565 if (iscalled) return;
1566 iscalled = kTRUE;
1567 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1568 Int_t ntosubmit = 0;
1569 TGridResult *res;
1570 TString jobID = "";
1571 if (!fNsubmitted) ntosubmit = 1;
1572 else {
1573 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
923e2ca5 1574 Printf("=== master %d: %s", lastmaster, status.Data());
d2a409b2 1575 // If last master not split, just return
1576 if (status != "SPLIT") {iscalled = kFALSE; return;}
1577 // No more than 100 waiting jobs
1578 if (nwaiting>100) {iscalled = kFALSE; return;}
1579 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1580 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
923e2ca5 1581 Printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs",
d2a409b2 1582 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1583 }
1584 Int_t nmasterjobs = fInputFiles->GetEntries();
1585 for (Int_t i=0; i<ntosubmit; i++) {
1586 // Submit for a range of enumeration of runs.
1587 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1588 TString query;
1589 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
923e2ca5 1590 Printf("********* %s",query.Data());
d2a409b2 1591 res = gGrid->Command(query);
1592 if (res) {
98ca124f 1593 TString cjobId1 = res->GetKey(0,"jobId");
1594 if (!cjobId1.Length()) {
d2a409b2 1595 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1596 iscalled = kFALSE;
1597 return;
1598 } else {
1599 Info("StartAnalysis", "\n_______________________________________________________________________ \
1600 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1601 \n_______________________________________________________________________",
98ca124f 1602 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1603 jobID += cjobId1;
1604 jobID += " ";
98ca124f 1605 lastmaster = cjobId1.Atoi();
d2a409b2 1606 if (!firstmaster) firstmaster = lastmaster;
1607 fNsubmitted++;
1608 }
1609 delete res;
1610 }
1611 }
1612 iscalled = kFALSE;
1613}
1614
c57f56b7 1615//______________________________________________________________________________
1616void AliAnalysisAlien::WriteAnalysisFile()
1617{
f10e8481 1618// Write current analysis manager into the file <analysisFile>
1619 TString analysisFile = fExecutable;
1620 analysisFile.ReplaceAll(".sh", ".root");
c57f56b7 1621 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1622 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1623 if (!mgr || !mgr->IsInitialized()) {
1624 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1625 return;
1626 }
1627 // Check analysis type
1628 TObject *handler;
1629 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1630 handler = (TObject*)mgr->GetInputEventHandler();
1631 if (handler) {
1632 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1633 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1634 }
1635 TDirectory *cdir = gDirectory;
f10e8481 1636 TFile *file = TFile::Open(analysisFile, "RECREATE");
c57f56b7 1637 if (file) {
1638 mgr->Write();
1639 delete file;
1640 }
1641 if (cdir) cdir->cd();
f10e8481 1642 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
c57f56b7 1643 }
1644 Bool_t copy = kTRUE;
1645 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1646 if (copy) {
1647 CdWork();
1648 TString workdir = gGrid->GetHomeDirectory();
1649 workdir += fGridWorkingDir;
f10e8481 1650 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1651 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1652 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
c57f56b7 1653 }
1654}
1655
1656//______________________________________________________________________________
1657void AliAnalysisAlien::WriteAnalysisMacro()
1658{
1659// Write the analysis macro that will steer the analysis in grid mode.
1660 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1661 ofstream out;
1662 out.open(fAnalysisMacro.Data(), ios::out);
1663 if (!out.good()) {
1664 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1665 return;
1666 }
1667 TString func = fAnalysisMacro;
1668 TString type = "ESD";
1669 TString comment = "// Analysis using ";
1670 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1671 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1672 type = "AOD";
1673 comment += "AOD";
1674 }
0df6ccf2 1675 if (type!="AOD" && fFriendChainName!="") {
1676 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1677 return;
1678 }
c57f56b7 1679 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1680 else comment += " data";
1681 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1682 func.ReplaceAll(".C", "");
1683 out << "void " << func.Data() << "()" << endl;
1684 out << "{" << endl;
1685 out << comment.Data() << endl;
1686 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1687 out << " TStopwatch timer;" << endl;
1688 out << " timer.Start();" << endl << endl;
c57f56b7 1689 out << "// load base root libraries" << endl;
1690 out << " gSystem->Load(\"libTree\");" << endl;
1691 out << " gSystem->Load(\"libGeom\");" << endl;
1692 out << " gSystem->Load(\"libVMC\");" << endl;
1693 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1694 out << "// Load analysis framework libraries" << endl;
4e5c5506 1695 if (!fPackages) {
4e5c5506 1696 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1697 out << " gSystem->Load(\"libESD\");" << endl;
1698 out << " gSystem->Load(\"libAOD\");" << endl;
1699 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1700 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1701 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1702 } else {
4e5c5506 1703 TIter next(fPackages);
1704 TObject *obj;
57377eb5 1705 TString pkgname;
1706 Bool_t hasSTEERBase = kFALSE;
1707 Bool_t hasESD = kFALSE;
1708 Bool_t hasAOD = kFALSE;
1709 Bool_t hasANALYSIS = kFALSE;
1710 Bool_t hasANALYSISalice = kFALSE;
1711 Bool_t hasCORRFW = kFALSE;
1712 while ((obj=next())) {
1713 pkgname = obj->GetName();
4478e6f1 1714 if (pkgname == "STEERBase" ||
1715 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1716 if (pkgname == "ESD" ||
1717 pkgname == "ESD.par") hasESD = kTRUE;
1718 if (pkgname == "AOD" ||
1719 pkgname == "AOD.par") hasAOD = kTRUE;
1720 if (pkgname == "ANALYSIS" ||
1721 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1722 if (pkgname == "ANALYSISalice" ||
1723 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1724 if (pkgname == "CORRFW" ||
1725 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1726 }
1727 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1728 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1729 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1730 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1731 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1732 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1733 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1734 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1735 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1736 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1737 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1738 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1739 out << "// Compile other par packages" << endl;
1740 next.Reset();
fcc9bb6f 1741 while ((obj=next())) {
1742 pkgname = obj->GetName();
4478e6f1 1743 if (pkgname == "STEERBase" ||
1744 pkgname == "STEERBase.par" ||
1745 pkgname == "ESD" ||
1746 pkgname == "ESD.par" ||
1747 pkgname == "AOD" ||
1748 pkgname == "AOD.par" ||
1749 pkgname == "ANALYSIS" ||
1750 pkgname == "ANALYSIS.par" ||
1751 pkgname == "ANALYSISalice" ||
1752 pkgname == "ANALYSISalice.par" ||
1753 pkgname == "CORRFW" ||
1754 pkgname == "CORRFW.par") continue;
4e5c5506 1755 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1756 }
4e5c5506 1757 }
43da816a 1758 out << "// include path" << endl;
1759 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1760 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1761 if (fAdditionalLibs.Length()) {
1762 out << "// Add aditional AliRoot libraries" << endl;
1763 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1764 TIter next(list);
1765 TObjString *str;
1766 while((str=(TObjString*)next())) {
1767 if (str->GetString().Contains(".so"))
1768 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1769 }
1770 if (list) delete list;
1771 }
1772 out << endl;
c57f56b7 1773 out << "// analysis source to be compiled at runtime (if any)" << endl;
1774 if (fAnalysisSource.Length()) {
1775 TObjArray *list = fAnalysisSource.Tokenize(" ");
1776 TIter next(list);
1777 TObjString *str;
1778 while((str=(TObjString*)next())) {
1779 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1780 }
1781 if (list) delete list;
1782 }
1783 out << endl;
1784 out << "// connect to AliEn and make the chain" << endl;
1785 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1786 if (IsUsingTags()) {
1787 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1788 } else {
76535c98 1789 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1790 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1791 } else {
1792 out << " // Check if the macro to create the chain was provided" << endl;
1793 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1794 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1795 out << " return;" << endl;
1796 out << " }" << endl;
1797 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1798 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1799 }
c57f56b7 1800 }
1801 out << "// read the analysis manager from file" << endl;
f10e8481 1802 TString analysisFile = fExecutable;
1803 analysisFile.ReplaceAll(".sh", ".root");
1804 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
c57f56b7 1805 out << " if (!file) return;" << endl;
1806 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1807 out << " AliAnalysisManager *mgr = 0;" << endl;
1808 out << " TKey *key;" << endl;
1809 out << " while ((key=(TKey*)nextkey())) {" << endl;
1810 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1811 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1812 out << " };" << endl;
1813 out << " if (!mgr) {" << endl;
f10e8481 1814 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
c57f56b7 1815 out << " return;" << endl;
1816 out << " }" << endl << endl;
1817 out << " mgr->PrintStatus();" << endl;
1818 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1819 out << " timer.Stop();" << endl;
1820 out << " timer.Print();" << endl;
c57f56b7 1821 out << "}" << endl << endl;
1822 if (IsUsingTags()) {
1823 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1824 out << "{" << endl;
1825 out << "// Create a chain using tags from the xml file." << endl;
1826 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1827 out << " if (!coll) {" << endl;
1828 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1829 out << " return NULL;" << endl;
1830 out << " }" << endl;
1831 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1832 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1833 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1834 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1835 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1836 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1837 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1838 out << " // Check if the cuts configuration file was provided" << endl;
1839 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1840 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1841 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1842 out << " }" << endl;
0df6ccf2 1843 if (fFriendChainName=="") {
1844 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1845 } else {
1846 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1847 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1848 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1849 }
c57f56b7 1850 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1851 out << " chain->ls();" << endl;
1852 out << " return chain;" << endl;
fcc9bb6f 1853 out << "}" << endl << endl;
c57f56b7 1854 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1855 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1856 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1857 msg += " AliLHCTagCuts *lhcCuts,\n";
1858 msg += " AliDetectorTagCuts *detCuts,\n";
1859 msg += " AliEventTagCuts *evCuts)";
1860 Info("WriteAnalysisMacro", msg.Data());
1861 }
0df6ccf2 1862 }
1863 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1864 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1865 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1866 out << "{" << endl;
1867 out << "// Create a chain using url's from xml file" << endl;
1868 out << " TString treename = type;" << endl;
1869 out << " treename.ToLower();" << endl;
1870 out << " treename += \"Tree\";" << endl;
e02fee64 1871 out << " printf(\"***************************************\\n\");" << endl;
1872 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1873 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1874 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1875 out << " if (!coll) {" << endl;
1876 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1877 out << " return NULL;" << endl;
1878 out << " }" << endl;
1879 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1880 if(fFriendChainName!="") {
1881 out << " TChain *chainFriend = new TChain(treename);" << endl;
1882 }
c57f56b7 1883 out << " coll->Reset();" << endl;
0df6ccf2 1884 out << " while (coll->Next()) {" << endl;
1885 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1886 if(fFriendChainName!="") {
1887 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1888 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1889 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1890 out << " chainFriend->Add(fileFriend.Data());" << endl;
1891 }
1892 out << " }" << endl;
c57f56b7 1893 out << " if (!chain->GetNtrees()) {" << endl;
1894 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1895 out << " return NULL;" << endl;
1896 out << " }" << endl;
0df6ccf2 1897 if(fFriendChainName!="") {
1898 out << " chain->AddFriend(chainFriend);" << endl;
1899 }
c57f56b7 1900 out << " return chain;" << endl;
fcc9bb6f 1901 out << "}" << endl << endl;
c57f56b7 1902 }
4e5c5506 1903 if (fPackages) {
fcc9bb6f 1904 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1905 out << "Bool_t SetupPar(const char *package) {" << endl;
1906 out << "// Compile the package and set it up." << endl;
1907 out << " TString pkgdir = package;" << endl;
1908 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1909 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1910 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1911 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1912 out << " // Check for BUILD.sh and execute" << endl;
1913 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1914 out << " printf(\"*******************************\\n\");" << endl;
1915 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1916 out << " printf(\"*******************************\\n\");" << endl;
1917 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1918 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1919 out << " gSystem->ChangeDirectory(cdir);" << endl;
1920 out << " return kFALSE;" << endl;
1921 out << " }" << endl;
1922 out << " } else {" << endl;
fcc9bb6f 1923 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1924 out << " gSystem->ChangeDirectory(cdir);" << endl;
1925 out << " return kFALSE;" << endl;
1926 out << " }" << endl;
1927 out << " // Check for SETUP.C and execute" << endl;
1928 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1929 out << " printf(\"*******************************\\n\");" << endl;
1930 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1931 out << " printf(\"*******************************\\n\");" << endl;
1932 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1933 out << " } else {" << endl;
fcc9bb6f 1934 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1935 out << " gSystem->ChangeDirectory(cdir);" << endl;
1936 out << " return kFALSE;" << endl;
1937 out << " }" << endl;
1938 out << " // Restore original workdir" << endl;
1939 out << " gSystem->ChangeDirectory(cdir);" << endl;
1940 out << " return kTRUE;" << endl;
1941 out << "}" << endl;
1942 }
c57f56b7 1943 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1944 }
1945 Bool_t copy = kTRUE;
1946 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1947 if (copy) {
1948 CdWork();
1949 TString workdir = gGrid->GetHomeDirectory();
1950 workdir += fGridWorkingDir;
1951 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1952 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1953 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1954 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1955 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1956 }
1957 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1958 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1959 }
1960}
1961
1962//______________________________________________________________________________
1963void AliAnalysisAlien::WriteExecutable()
1964{
1965// Generate the alien executable script.
1966 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1967 ofstream out;
1968 out.open(fExecutable.Data(), ios::out);
1969 if (out.bad()) {
5513444a 1970 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1971 return;
1972 }
1973 out << "#!/bin/bash" << endl;
c57f56b7 1974 out << "echo \"=========================================\"" << endl;
1975 out << "echo \"############## PATH : ##############\"" << endl;
1976 out << "echo $PATH" << endl;
1977 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1978 out << "echo $LD_LIBRARY_PATH" << endl;
1979 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1980 out << "echo $ROOTSYS" << endl;
1981 out << "echo \"############## which root : ##############\"" << endl;
1982 out << "which root" << endl;
1983 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1984 out << "echo $ALICE_ROOT" << endl;
1985 out << "echo \"############## which aliroot : ##############\"" << endl;
1986 out << "which aliroot" << endl;
1987 out << "echo \"=========================================\"" << endl << endl;
1988// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
0a1c1f7f 1989 out << fExecutableCommand << " ";
c57f56b7 1990 out << fAnalysisMacro.Data() << endl << endl;
1991 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1992 }
1993 Bool_t copy = kTRUE;
1994 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1995 if (copy) {
1996 CdWork();
1997 TString workdir = gGrid->GetHomeDirectory();
923e2ca5 1998 TString bindir = Form("%s/bin", workdir.Data());
1999 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
c57f56b7 2000 workdir += fGridWorkingDir;
2001 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2002 if (FileExists(executable)) gGrid->Rm(executable);
2003 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2004 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2005 }
2006}
2007
2008//______________________________________________________________________________
5513444a 2009void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2010{
2011// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 2012// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 2013// Next lines: full_path_to_dataset XXX (XXX is a string)
2014// To submit, one has to: submit jdl XXX for all lines
2015 ofstream out;
2016 out.open(filename, ios::out);
2017 if (out.bad()) {
2018 Error("WriteProductionFile", "Bad file name: %s", filename);
2019 return;
2020 }
2021 TString workdir = gGrid->GetHomeDirectory();
2022 workdir += fGridWorkingDir;
f5e8c702 2023 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 2024 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 2025 out << locjdl << " " << njobspermaster << endl;
5513444a 2026 Int_t nmasterjobs = fInputFiles->GetEntries();
2027 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 2028 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 2029 }
2030 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2031 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2032}
2033
2034//______________________________________________________________________________
c57f56b7 2035void AliAnalysisAlien::WriteValidationScript()
2036{
2037// Generate the alien validation script.
2038 // Generate the validation script
2039 TObjString *os;
0d5d317c 2040 TString validationScript = fExecutable;
2041 validationScript.ReplaceAll(".sh", "_validation.sh");
c57f56b7 2042 if (!Connect()) {
2043 Error("WriteValidationScript", "Alien connection required");
2044 return;
2045 }
2046 TString out_stream = "";
2047 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2048 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2049 ofstream out;
0d5d317c 2050 out.open(validationScript, ios::out);
c57f56b7 2051 out << "#!/bin/bash" << endl;
2052 out << "##################################################" << endl;
2053 out << "validateout=`dirname $0`" << endl;
2054 out << "validatetime=`date`" << endl;
2055 out << "validated=\"0\";" << endl;
2056 out << "error=0" << endl;
2057 out << "if [ -z $validateout ]" << endl;
2058 out << "then" << endl;
2059 out << " validateout=\".\"" << endl;
2060 out << "fi" << endl << endl;
2061 out << "cd $validateout;" << endl;
2062 out << "validateworkdir=`pwd`;" << endl << endl;
2063 out << "echo \"*******************************************************\"" << out_stream << endl;
2064 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2065 out << "" << endl;
2066 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2067 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2068 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2069 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2070 out << "ls -la ./" << out_stream << endl;
2071 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2072 out << "##################################################" << endl;
ebec370a 2073
2074 out << "" << endl;
2075 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2076 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2077 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2078 out << "" << endl;
2079
2080 out << "if [ ! -f stderr ] ; then" << endl;
2081 out << " error=1" << endl;
2082 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2083 out << " echo \"Error = $error\" " << out_stream << endl;
2084 out << "fi" << endl;
2085
2086 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2087 out << " error=1" << endl;
2088 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2089 out << " echo \"$parArch\" " << out_stream << endl;
2090 out << " echo \"Error = $error\" " << out_stream << endl;
2091 out << "fi" << endl;
2092
2093 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2094 out << " error=1" << endl;
2095 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2096 out << " echo \"$segViol\" " << out_stream << endl;
2097 out << " echo \"Error = $error\" " << out_stream << endl;
2098 out << "fi" << endl;
2099
2100 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2101 out << " error=1" << endl;
2102 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2103 out << " echo \"$segFault\" " << out_stream << endl;
2104 out << " echo \"Error = $error\" " << out_stream << endl;
2105 out << "fi" << endl;
2106
2107 // Part dedicated to the specific analyses running into the train
2108
c57f56b7 2109 TObjArray *arr = fOutputFiles.Tokenize(" ");
2110 TIter next1(arr);
2111 TString output_file;
2112 while ((os=(TObjString*)next1())) {
2113 output_file = os->GetString();
2114 Int_t index = output_file.Index("@");
2115 if (index > 0) output_file.Remove(index);
2116 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2117 out << " error=1" << endl;
2118 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2119 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2120 out << "fi" << endl;
2121 }
2122 delete arr;
923e2ca5 2123 out << "if ! [ -f outputs_valid ] ; then" << endl;
2124 out << " error=1" << endl;
2125 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2126 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2127 out << "fi" << endl;
2128
c57f56b7 2129 out << "if [ $error = 0 ] ; then" << endl;
2130 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2131 out << "fi" << endl;
2132
2133 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2134 out << "echo \"*******************************************************\"" << out_stream << endl;
2135 out << "cd -" << endl;
2136 out << "exit $error" << endl;
2137 }
2138 Bool_t copy = kTRUE;
2139 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2140 if (copy) {
2141 CdWork();
2142 TString workdir = gGrid->GetHomeDirectory();
2143 workdir += fGridWorkingDir;
0d5d317c 2144 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2145 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2146 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
c57f56b7 2147 }
2148}