Protection against crash when attempting to fit empty histos
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
CommitLineData
c57f56b7 1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
d2a409b2 33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
c57f56b7 35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
bb885a9e 37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
c57f56b7 39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
bb885a9e 52 fNtestFiles(0),
319593fb 53 fNrunsPerMaster(0),
16a4353c 54 fMaxMergeFiles(0),
d2a409b2 55 fNsubmitted(0),
a3e84053 56 fProductionMode(0),
c57f56b7 57 fRunNumbers(),
58 fExecutable(),
59 fArguments(),
60 fAnalysisMacro(),
61 fAnalysisSource(),
62 fAdditionalLibs(),
63 fSplitMode(),
64 fAPIVersion(),
65 fROOTVersion(),
66 fAliROOTVersion(),
67 fUser(),
68 fGridWorkingDir(),
69 fGridDataDir(),
70 fDataPattern(),
71 fGridOutputDir(),
72 fOutputArchive(),
73 fOutputFiles(),
74 fInputFormat(),
e7c71df0 75 fDatasetName(),
c57f56b7 76 fJDLName(),
bb885a9e 77 fMergeExcludes(),
f965131e 78 fIncludePath(),
bb885a9e 79 fCloseSE(),
0df6ccf2 80 fFriendChainName(),
4e5c5506 81 fInputFiles(0),
82 fPackages(0)
c57f56b7 83{
84// Dummy ctor.
85 SetDefaults();
86}
87
88//______________________________________________________________________________
89AliAnalysisAlien::AliAnalysisAlien(const char *name)
90 :AliAnalysisGrid(name),
91 fGridJDL(NULL),
92 fPrice(0),
93 fTTL(0),
94 fSplitMaxInputFileNumber(0),
95 fMaxInitFailed(0),
96 fMasterResubmitThreshold(0),
bb885a9e 97 fNtestFiles(0),
319593fb 98 fNrunsPerMaster(0),
16a4353c 99 fMaxMergeFiles(0),
d2a409b2 100 fNsubmitted(0),
a3e84053 101 fProductionMode(0),
c57f56b7 102 fRunNumbers(),
103 fExecutable(),
104 fArguments(),
105 fAnalysisMacro(),
106 fAnalysisSource(),
107 fAdditionalLibs(),
108 fSplitMode(),
109 fAPIVersion(),
110 fROOTVersion(),
111 fAliROOTVersion(),
112 fUser(),
113 fGridWorkingDir(),
114 fGridDataDir(),
115 fDataPattern(),
116 fGridOutputDir(),
117 fOutputArchive(),
118 fOutputFiles(),
119 fInputFormat(),
e7c71df0 120 fDatasetName(),
c57f56b7 121 fJDLName(),
bb885a9e 122 fMergeExcludes(),
f965131e 123 fIncludePath(),
bb885a9e 124 fCloseSE(),
0df6ccf2 125 fFriendChainName(),
4e5c5506 126 fInputFiles(0),
127 fPackages(0)
c57f56b7 128{
129// Default ctor.
130 SetDefaults();
131}
132
133//______________________________________________________________________________
134AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
135 :AliAnalysisGrid(other),
136 fGridJDL(NULL),
137 fPrice(other.fPrice),
138 fTTL(other.fTTL),
139 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
140 fMaxInitFailed(other.fMaxInitFailed),
141 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
bb885a9e 142 fNtestFiles(other.fNtestFiles),
319593fb 143 fNrunsPerMaster(other.fNrunsPerMaster),
16a4353c 144 fMaxMergeFiles(other.fMaxMergeFiles),
d2a409b2 145 fNsubmitted(other.fNsubmitted),
a3e84053 146 fProductionMode(other.fProductionMode),
c57f56b7 147 fRunNumbers(other.fRunNumbers),
148 fExecutable(other.fExecutable),
149 fArguments(other.fArguments),
150 fAnalysisMacro(other.fAnalysisMacro),
151 fAnalysisSource(other.fAnalysisSource),
152 fAdditionalLibs(other.fAdditionalLibs),
153 fSplitMode(other.fSplitMode),
154 fAPIVersion(other.fAPIVersion),
155 fROOTVersion(other.fROOTVersion),
156 fAliROOTVersion(other.fAliROOTVersion),
157 fUser(other.fUser),
158 fGridWorkingDir(other.fGridWorkingDir),
159 fGridDataDir(other.fGridDataDir),
160 fDataPattern(other.fDataPattern),
161 fGridOutputDir(other.fGridOutputDir),
162 fOutputArchive(other.fOutputArchive),
163 fOutputFiles(other.fOutputFiles),
164 fInputFormat(other.fInputFormat),
e7c71df0 165 fDatasetName(other.fDatasetName),
c57f56b7 166 fJDLName(other.fJDLName),
bb885a9e 167 fMergeExcludes(other.fMergeExcludes),
f965131e 168 fIncludePath(other.fIncludePath),
bb885a9e 169 fCloseSE(other.fCloseSE),
0df6ccf2 170 fFriendChainName(other.fFriendChainName),
4e5c5506 171 fInputFiles(0),
172 fPackages(0)
c57f56b7 173{
174// Copy ctor.
175 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
a8739e8a 176 fRunRange[0] = other.fRunRange[0];
177 fRunRange[1] = other.fRunRange[1];
c57f56b7 178 if (other.fInputFiles) {
179 fInputFiles = new TObjArray();
180 TIter next(other.fInputFiles);
181 TObject *obj;
182 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
183 fInputFiles->SetOwner();
184 }
4e5c5506 185 if (other.fPackages) {
186 fPackages = new TObjArray();
187 TIter next(other.fPackages);
188 TObject *obj;
189 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
190 fPackages->SetOwner();
191 }
c57f56b7 192}
193
194//______________________________________________________________________________
195AliAnalysisAlien::~AliAnalysisAlien()
196{
197// Destructor.
198 if (fGridJDL) delete fGridJDL;
199 if (fInputFiles) delete fInputFiles;
4e5c5506 200 if (fPackages) delete fPackages;
c57f56b7 201}
202
203//______________________________________________________________________________
204AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
205{
206// Assignment.
207 if (this != &other) {
208 AliAnalysisGrid::operator=(other);
209 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
210 fPrice = other.fPrice;
211 fTTL = other.fTTL;
212 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
213 fMaxInitFailed = other.fMaxInitFailed;
214 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
bb885a9e 215 fNtestFiles = other.fNtestFiles;
a3e84053 216 fNrunsPerMaster = other.fNrunsPerMaster;
217 fMaxMergeFiles = other.fMaxMergeFiles;
218 fNsubmitted = other.fNsubmitted;
219 fProductionMode = other.fProductionMode;
c57f56b7 220 fRunNumbers = other.fRunNumbers;
221 fExecutable = other.fExecutable;
222 fArguments = other.fArguments;
223 fAnalysisMacro = other.fAnalysisMacro;
224 fAnalysisSource = other.fAnalysisSource;
225 fAdditionalLibs = other.fAdditionalLibs;
226 fSplitMode = other.fSplitMode;
227 fAPIVersion = other.fAPIVersion;
228 fROOTVersion = other.fROOTVersion;
229 fAliROOTVersion = other.fAliROOTVersion;
230 fUser = other.fUser;
231 fGridWorkingDir = other.fGridWorkingDir;
232 fGridDataDir = other.fGridDataDir;
233 fDataPattern = other.fDataPattern;
234 fGridOutputDir = other.fGridOutputDir;
235 fOutputArchive = other.fOutputArchive;
236 fOutputFiles = other.fOutputFiles;
237 fInputFormat = other.fInputFormat;
e7c71df0 238 fDatasetName = other.fDatasetName;
c57f56b7 239 fJDLName = other.fJDLName;
bb885a9e 240 fMergeExcludes = other.fMergeExcludes;
f965131e 241 fIncludePath = other.fIncludePath;
bb885a9e 242 fCloseSE = other.fCloseSE;
0df6ccf2 243 fFriendChainName = other.fFriendChainName;
c57f56b7 244 if (other.fInputFiles) {
245 fInputFiles = new TObjArray();
246 TIter next(other.fInputFiles);
247 TObject *obj;
248 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
249 fInputFiles->SetOwner();
250 }
4e5c5506 251 if (other.fPackages) {
252 fPackages = new TObjArray();
253 TIter next(other.fPackages);
254 TObject *obj;
255 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
256 fPackages->SetOwner();
257 }
c57f56b7 258 }
259 return *this;
260}
261
262//______________________________________________________________________________
f965131e 263void AliAnalysisAlien::AddIncludePath(const char *path)
264{
265// Add include path in the remote analysis macro.
266 TString p(path);
267 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
268 else fIncludePath += Form("-I%s ", path);
269}
270
271//______________________________________________________________________________
c57f56b7 272void AliAnalysisAlien::AddRunNumber(Int_t run)
273{
274// Add a run number to the list of runs to be processed.
275 if (fRunNumbers.Length()) fRunNumbers += " ";
276 fRunNumbers += Form("%d", run);
277}
278
279//______________________________________________________________________________
280void AliAnalysisAlien::AddDataFile(const char *lfn)
281{
282// Adds a data file to the input to be analysed. The file should be a valid LFN
283// or point to an existing file in the alien workdir.
284 if (!fInputFiles) fInputFiles = new TObjArray();
285 fInputFiles->Add(new TObjString(lfn));
286}
287
288//______________________________________________________________________________
289Bool_t AliAnalysisAlien::Connect()
290{
291// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
292 if (gGrid && gGrid->IsConnected()) return kTRUE;
293 if (!gSystem->Getenv("alien_API_USER")) {
294 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
295 gSystem->Getenv("UID"));
296 return kFALSE;
297 }
298 if (!gGrid) {
299 Info("Connect", "Trying to connect to AliEn ...");
300 TGrid::Connect("alien://");
301 }
302 if (!gGrid || !gGrid->IsConnected()) {
303 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
304 return kFALSE;
305 }
306 fUser = gGrid->GetUser();
307 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
308 return kTRUE;
309}
310
311//______________________________________________________________________________
312void AliAnalysisAlien::CdWork()
313{
314// Check validity of alien workspace. Create directory if possible.
315 if (!Connect()) {
316 Error("CdWork", "Alien connection required");
317 return;
318 }
319 TString homedir = gGrid->GetHomeDirectory();
320 TString workdir = homedir + fGridWorkingDir;
321 if (!gGrid->Cd(workdir)) {
322 gGrid->Cd(homedir);
323 if (gGrid->Mkdir(workdir)) {
324 gGrid->Cd(fGridWorkingDir);
325 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
326 } else {
327 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
328 workdir.Data(), homedir.Data());
329 fGridWorkingDir = "";
330 }
331 }
332}
333
334//______________________________________________________________________________
335Bool_t AliAnalysisAlien::CheckInputData()
336{
337// Check validity of input data. If necessary, create xml files.
a8739e8a 338 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
d2a409b2 339 if (!fGridDataDir.Length()) {
340 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
341 return kFALSE;
342 }
343 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
344 return kTRUE;
c57f56b7 345 }
346 // Process declared files
347 Bool_t is_collection = kFALSE;
348 Bool_t is_xml = kFALSE;
349 Bool_t use_tags = kFALSE;
350 Bool_t checked = kFALSE;
351 CdWork();
352 TString file;
353 TString workdir = gGrid->GetHomeDirectory();
354 workdir += fGridWorkingDir;
355 if (fInputFiles) {
356 TObjString *objstr;
357 TIter next(fInputFiles);
358 while ((objstr=(TObjString*)next())) {
359 file = workdir;
360 file += "/";
361 file += objstr->GetString();
362 // Store full lfn path
363 if (FileExists(file)) objstr->SetString(file);
364 else {
365 file = objstr->GetName();
366 if (!FileExists(objstr->GetName())) {
367 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
368 objstr->GetName(), workdir.Data());
369 return kFALSE;
370 }
371 }
372 Bool_t iscoll, isxml, usetags;
373 CheckDataType(file, iscoll, isxml, usetags);
374 if (!checked) {
375 checked = kTRUE;
376 is_collection = iscoll;
377 is_xml = isxml;
378 use_tags = usetags;
379 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
380 } else {
381 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
382 Error("CheckInputData", "Some conflict was found in the types of inputs");
383 return kFALSE;
384 }
385 }
386 }
387 }
388 // Process requested run numbers
a8739e8a 389 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
c57f56b7 390 // Check validity of alien data directory
391 if (!fGridDataDir.Length()) {
392 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
393 return kFALSE;
394 }
395 if (!gGrid->Cd(fGridDataDir)) {
396 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
397 return kFALSE;
398 }
399 if (is_collection) {
400 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
401 return kFALSE;
402 }
403
404 if (checked && !is_xml) {
405 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
406 return kFALSE;
407 }
408 // Check validity of run number(s)
409 TObjArray *arr;
410 TObjString *os;
319593fb 411 Int_t nruns = 0;
412 TString schunk;
c57f56b7 413 TString path;
414 if (!checked) {
415 checked = kTRUE;
416 use_tags = fDataPattern.Contains("tag");
417 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
418 }
419 if (use_tags != fDataPattern.Contains("tag")) {
420 Error("CheckInputData", "Cannot mix input files using/not using tags");
421 return kFALSE;
422 }
423 if (fRunNumbers.Length()) {
a8739e8a 424 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
c57f56b7 425 arr = fRunNumbers.Tokenize(" ");
426 TIter next(arr);
427 while ((os=(TObjString*)next())) {
428 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
429 if (!gGrid->Cd(path)) {
a8739e8a 430 Warning("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
431 continue;
c57f56b7 432 }
433 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
434 TString msg = "\n##### file: ";
435 msg += path;
436 msg += " type: xml_collection;";
437 if (use_tags) msg += " using_tags: Yes";
438 else msg += " using_tags: No";
439 Info("CheckDataType", msg.Data());
319593fb 440 if (fNrunsPerMaster<2) {
d2a409b2 441 AddDataFile(Form("%s.xml", os->GetString().Data()));
319593fb 442 } else {
443 nruns++;
444 if (((nruns-1)%fNrunsPerMaster) == 0) {
445 schunk = os->GetString();
446 }
447 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
448 schunk += Form("_%s.xml", os->GetString().Data());
d2a409b2 449 AddDataFile(schunk);
319593fb 450 }
c57f56b7 451 }
452 delete arr;
a8739e8a 453 } else {
454 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
455 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
456 path = Form("%s/%d ", fGridDataDir.Data(), irun);
457 if (!gGrid->Cd(path)) {
458 Warning("CheckInputData", "Run number %d not found in path: %s", irun, path.Data());
459 continue;
460 }
461 path = Form("%s/%d.xml", workdir.Data(),irun);
462 TString msg = "\n##### file: ";
463 msg += path;
464 msg += " type: xml_collection;";
465 if (use_tags) msg += " using_tags: Yes";
466 else msg += " using_tags: No";
467 Info("CheckDataType", msg.Data());
319593fb 468 if (fNrunsPerMaster<2) {
d2a409b2 469 AddDataFile(Form("%d.xml",irun));
319593fb 470 } else {
471 nruns++;
472 if (((nruns-1)%fNrunsPerMaster) == 0) {
473 schunk = Form("%d", irun);
474 }
475 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
476 schunk += Form("_%d.xml", irun);
d2a409b2 477 AddDataFile(schunk);
319593fb 478 }
a8739e8a 479 }
c57f56b7 480 }
481 return kTRUE;
482}
483
484//______________________________________________________________________________
485Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
486{
487// Create dataset for the grid data directory + run number.
488 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
489 if (!Connect()) {
490 Error("CreateDataset", "Cannot create dataset with no grid connection");
491 return kFALSE;
492 }
493
494 // Cd workspace
495 CdWork();
496 TString workdir = gGrid->GetHomeDirectory();
497 workdir += fGridWorkingDir;
498
499 // Compose the 'find' command arguments
500 TString command;
501 TString options = "-x collection ";
bb885a9e 502 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
c57f56b7 503 TString conditions = "";
504
505 TString file;
506 TString path;
319593fb 507 Int_t nruns = 0;
508 TString schunk;
ab254fd1 509 TGridCollection *cbase=0, *cadd=0;
d2a409b2 510 if (!fRunNumbers.Length() && !fRunRange[0]) {
511 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
512 // Make a single data collection from data directory.
513 path = fGridDataDir;
514 if (!gGrid->Cd(path)) {
515 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
516 return kFALSE;
517 }
518 CdWork();
519 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
520 else file = Form("%s.xml", gSystem->BaseName(path));
521 if (gSystem->AccessPathName(file)) {
522 command = "find ";
523 command += options;
524 command += path;
525 command += " ";
526 command += pattern;
527 command += conditions;
528 printf("command: %s\n", command.Data());
529 TGridResult *res = gGrid->Command(command);
530 if (res) delete res;
531 // Write standard output to file
532 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
533 }
534 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
535 // Copy xml file to alien space
536 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
537 if (!FileExists(file)) {
538 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
539 return kFALSE;
540 }
541 // Update list of files to be processed.
542 }
543 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
544 return kTRUE;
545 }
c57f56b7 546 // Several runs
a8739e8a 547 if (fRunNumbers.Length()) {
548 TObjArray *arr = fRunNumbers.Tokenize(" ");
549 TObjString *os;
550 TIter next(arr);
551 while ((os=(TObjString*)next())) {
552 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
673d4fb3 553 if (!gGrid->Cd(path)) continue;
95e5b448 554 CdWork();
a8739e8a 555 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
556 else file = Form("%s.xml", os->GetString().Data());
319593fb 557 // If local collection file does not exist, create it via 'find' command.
558 if (gSystem->AccessPathName(file)) {
559 command = "find ";
560 command += options;
561 command += path;
562 command += pattern;
563 command += conditions;
564 TGridResult *res = gGrid->Command(command);
565 if (res) delete res;
566 // Write standard output to file
567 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
568 }
a8739e8a 569 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 570 // Check if there is one run per master job.
571 if (fNrunsPerMaster<2) {
572 if (FileExists(file)) {
573 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
574 continue;
575 }
576 // Copy xml file to alien space
577 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
578 if (!FileExists(file)) {
579 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
580 delete arr;
581 return kFALSE;
582 }
583 } else {
584 nruns++;
585 if (((nruns-1)%fNrunsPerMaster) == 0) {
586 schunk = os->GetString();
587 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
588 } else {
589 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
590 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
591 cbase->Add(cadd);
592 delete cadd;
593 }
594 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
595 continue;
596 }
597 schunk += Form("_%s.xml", os->GetString().Data());
598 if (FileExists(schunk)) {
599 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
600 continue;
601 }
602 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
603 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
604 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
605 if (!FileExists(schunk)) {
606 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
607 delete arr;
608 return kFALSE;
609 }
610 }
a8739e8a 611 }
612 delete arr;
613 } else {
614 // Process a full run range.
615 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
616 path = Form("%s/%d ", fGridDataDir.Data(), irun);
673d4fb3 617 if (!gGrid->Cd(path)) continue;
95e5b448 618 CdWork();
a8739e8a 619 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
620 else file = Form("%d.xml", irun);
319593fb 621 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
a8739e8a 622 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
623// gGrid->Rm(file);
624 continue;
625 }
319593fb 626 // If local collection file does not exist, create it via 'find' command.
627 if (gSystem->AccessPathName(file)) {
628 command = "find ";
629 command += options;
630 command += path;
631 command += pattern;
632 command += conditions;
633 TGridResult *res = gGrid->Command(command);
634 if (res) delete res;
635 // Write standard output to file
636 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
637 }
a8739e8a 638 if (TestBit(AliAnalysisGrid::kTest)) break;
319593fb 639 // Check if there is one run per master job.
640 if (fNrunsPerMaster<2) {
641 if (FileExists(file)) {
642 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
643 continue;
644 }
645 // Copy xml file to alien space
646 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
647 if (!FileExists(file)) {
648 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
649 return kFALSE;
650 }
651 } else {
652 nruns++;
95e5b448 653 // Check if the collection for the chunk exist locally.
654 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
655 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
319593fb 656 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
657 if (((nruns-1)%fNrunsPerMaster) == 0) {
658 schunk = Form("%d", irun);
659 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
660 } else {
661 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
662 cbase->Add(cadd);
663 delete cadd;
664 }
665 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1]) {
666 continue;
667 }
668 schunk += Form("_%d.xml", irun);
669 if (FileExists(schunk)) {
670 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
671 continue;
672 }
673 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
674 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
95e5b448 675 if (FileExists(schunk)) {
676 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
677 continue;
678 }
319593fb 679 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
680 if (!FileExists(schunk)) {
681 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
682 return kFALSE;
683 }
684 }
c57f56b7 685 }
a8739e8a 686 }
c57f56b7 687 return kTRUE;
688}
689
690//______________________________________________________________________________
691Bool_t AliAnalysisAlien::CreateJDL()
692{
693// Generate a JDL file according to current settings. The name of the file is
694// specified by fJDLName.
695 Bool_t error = kFALSE;
696 TObjArray *arr = 0;
697 Bool_t copy = kTRUE;
698 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
699 Bool_t generate = kTRUE;
700 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
701 if (!Connect()) {
702 Error("CreateJDL", "Alien connection required");
703 return kFALSE;
704 }
705 // Check validity of alien workspace
706 CdWork();
707 TString workdir = gGrid->GetHomeDirectory();
708 workdir += fGridWorkingDir;
709 if (generate) {
710 TObjString *os;
711 if (!fInputFiles) {
712 Error("CreateJDL()", "Define some input files for your analysis.");
713 error = kTRUE;
714 }
715 // Compose list of input files
716 // Check if output files were defined
717 if (!fOutputFiles.Length()) {
718 Error("CreateJDL", "You must define at least one output file");
719 error = kTRUE;
720 }
721 // Check if an output directory was defined and valid
722 if (!fGridOutputDir.Length()) {
723 Error("CreateJDL", "You must define AliEn output directory");
724 error = kTRUE;
725 } else {
d2a409b2 726 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
c57f56b7 727 if (!gGrid->Cd(fGridOutputDir)) {
728 if (gGrid->Mkdir(fGridOutputDir)) {
729 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
730 } else {
731 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
5513444a 732 // error = kTRUE;
c57f56b7 733 }
734 }
735 gGrid->Cd(workdir);
736 }
737 // Exit if any error up to now
738 if (error) return kFALSE;
739 // Set JDL fields
740 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
3c639fc4 741 fGridJDL->SetExecutable(fExecutable);
c57f56b7 742// fGridJDL->SetTTL((UInt_t)fTTL);
743 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
744 if (fMaxInitFailed > 0)
745 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
746 if (fSplitMaxInputFileNumber > 0)
747 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
748 if (fSplitMode.Length())
749 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
750// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
751 if (fAliROOTVersion.Length())
752 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
753 if (fROOTVersion.Length())
754 fGridJDL->AddToPackages("ROOT", fROOTVersion);
755 if (fAPIVersion.Length())
756 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
757 fGridJDL->SetInputDataListFormat(fInputFormat);
758 fGridJDL->SetInputDataList("wn.xml");
c57f56b7 759 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
760 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
761 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
762 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
763 if (fAdditionalLibs.Length()) {
764 arr = fAdditionalLibs.Tokenize(" ");
765 TIter next(arr);
766 while ((os=(TObjString*)next())) {
767 if (os->GetString().Contains(".so")) continue;
768 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
769 }
770 delete arr;
771 }
4e5c5506 772 if (fPackages) {
773 TIter next(fPackages);
774 TObject *obj;
775 while ((obj=next()))
776 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
777 }
c57f56b7 778 if (fOutputArchive.Length()) {
779 arr = fOutputArchive.Tokenize(" ");
780 TIter next(arr);
781 while ((os=(TObjString*)next()))
43da816a 782 if (!os->GetString().Contains("@") && fCloseSE.Length())
783 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
784 else
785 fGridJDL->AddToOutputArchive(os->GetString());
c57f56b7 786 delete arr;
787 }
c57f56b7 788 arr = fOutputFiles.Tokenize(" ");
789 TIter next(arr);
43da816a 790 while ((os=(TObjString*)next())) {
791 if (!os->GetString().Contains("@") && fCloseSE.Length())
792 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
793 else
794 fGridJDL->AddToOutputSandbox(os->GetString());
795 }
c57f56b7 796 delete arr;
797// fGridJDL->SetPrice((UInt_t)fPrice);
798 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
799 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
800 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
d2a409b2 801 // Write a jdl with 2 input parameters: collection name and output dir name.
802 WriteJDL(copy);
c57f56b7 803 }
804 // Copy jdl to grid workspace
a8739e8a 805 if (copy) {
c57f56b7 806 if (fAdditionalLibs.Length()) {
e7c71df0 807 arr = fAdditionalLibs.Tokenize(" ");
c57f56b7 808 TObjString *os;
809 TIter next(arr);
810 while ((os=(TObjString*)next())) {
c57f56b7 811 if (os->GetString().Contains(".so")) continue;
4e5c5506 812 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
c57f56b7 813 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
814 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
815 }
816 delete arr;
817 }
4e5c5506 818 if (fPackages) {
819 TIter next(fPackages);
820 TObject *obj;
821 while ((obj=next())) {
822 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
823 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
824 }
825 }
c57f56b7 826 }
827 return kTRUE;
828}
829
830//______________________________________________________________________________
d2a409b2 831Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
a8739e8a 832{
833// Writes one or more JDL's corresponding to findex. If findex is negative,
834// all run numbers are considered in one go (jdl). For non-negative indices
835// they correspond to the indices in the array fInputFiles.
836 if (!fInputFiles) return kFALSE;
837 TObjString *os;
a8739e8a 838 TString workdir = gGrid->GetHomeDirectory();
839 workdir += fGridWorkingDir;
d2a409b2 840
841 if (!fRunNumbers.Length() && !fRunRange[0]) {
842 // One jdl with no parameters in case input data is specified by name.
a8739e8a 843 TIter next(fInputFiles);
844 while ((os=(TObjString*)next()))
845 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
d2a409b2 846 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 847 } else {
d2a409b2 848 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
849 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
850 fGridJDL->SetOutputDirectory(Form("%s/$2#alien_counter_03i#", fGridOutputDir.Data()));
a8739e8a 851 }
852
853
854 // Generate the JDL as a string
855 TString sjdl = fGridJDL->Generate();
856 Int_t index;
857 index = sjdl.Index("Executable");
858 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
859 index = sjdl.Index("Split ");
860 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
861 index = sjdl.Index("SplitMaxInputFileNumber");
862 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
863 index = sjdl.Index("InputDataCollection");
864 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
a8739e8a 865 index = sjdl.Index("InputFile");
866 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
867 index = sjdl.Index("InputDataList ");
868 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
869 index = sjdl.Index("InputDataListFormat");
870 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
871 index = sjdl.Index("Price");
872 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
873 index = sjdl.Index("Requirements");
874 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
875 index = sjdl.Index("Packages");
876 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
877 index = sjdl.Index("User =");
878 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
879 index = sjdl.Index("TTL");
880 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
881 index = sjdl.Index("OutputFile");
882 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
883 index = sjdl.Index("OutputDir");
884 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
885 index = sjdl.Index("OutputArchive");
886 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
887 index = sjdl.Index("MaxInitFailed");
888 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
889 index = sjdl.Index("MasterResubmitThreshold");
890 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
891 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
892 index = sjdl.Index("Validationcommand");
893 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
894 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
895 sjdl.ReplaceAll("(member", "\n (member");
896 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
897 sjdl.ReplaceAll("{", "{\n ");
898 sjdl.ReplaceAll("};", "\n};");
899 sjdl.ReplaceAll("{\n \n", "{\n");
900 sjdl.ReplaceAll("\n\n", "\n");
901 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
902 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
903 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
904 index = sjdl.Index("JDLVariables");
905 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
906 // Write jdl to file
a8739e8a 907 ofstream out;
d2a409b2 908 out.open(fJDLName.Data(), ios::out);
a8739e8a 909 if (out.bad()) {
d2a409b2 910 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
a8739e8a 911 return kFALSE;
912 }
913 out << sjdl << endl;
914
915 // Copy jdl to grid workspace
916 if (!copy) {
917 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
918 } else {
d2a409b2 919 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
920 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
a3e84053 921 if (fProductionMode)
b5fe9cba 922 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
d2a409b2 923 if (FileExists(locjdl)) gGrid->Rm(locjdl);
5513444a 924 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
a8739e8a 925 }
926 return kTRUE;
927}
928
929//______________________________________________________________________________
5513444a 930Bool_t AliAnalysisAlien::FileExists(const char *lfn)
c57f56b7 931{
932// Returns true if file exists.
5513444a 933 if (!gGrid) return kFALSE;
c57f56b7 934 TGridResult *res = gGrid->Ls(lfn);
935 if (!res) return kFALSE;
936 TMap *map = dynamic_cast<TMap*>(res->At(0));
937 if (!map) {
938 delete res;
939 return kFALSE;
940 }
941 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
942 if (!objs || !objs->GetString().Length()) {
943 delete res;
944 return kFALSE;
945 }
946 delete res;
947 return kTRUE;
948}
949
950//______________________________________________________________________________
951void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
952{
953// Check input data type.
954 is_collection = kFALSE;
955 is_xml = kFALSE;
956 use_tags = kFALSE;
957 if (!gGrid) {
958 Error("CheckDataType", "No connection to grid");
959 return;
960 }
961 is_collection = IsCollection(lfn);
962 TString msg = "\n##### file: ";
963 msg += lfn;
964 if (is_collection) {
965 msg += " type: raw_collection;";
966 // special treatment for collections
967 is_xml = kFALSE;
968 // check for tag files in the collection
969 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
970 if (!res) {
971 msg += " using_tags: No (unknown)";
972 Info("CheckDataType", msg.Data());
973 return;
974 }
975 const char* typeStr = res->GetKey(0, "origLFN");
976 if (!typeStr || !strlen(typeStr)) {
977 msg += " using_tags: No (unknown)";
978 Info("CheckDataType", msg.Data());
979 return;
980 }
981 TString file = typeStr;
982 use_tags = file.Contains(".tag");
983 if (use_tags) msg += " using_tags: Yes";
984 else msg += " using_tags: No";
985 Info("CheckDataType", msg.Data());
986 return;
987 }
988 TString slfn(lfn);
989 slfn.ToLower();
990 is_xml = slfn.Contains(".xml");
991 if (is_xml) {
992 // Open xml collection and check if there are tag files inside
993 msg += " type: xml_collection;";
994 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
995 if (!coll) {
996 msg += " using_tags: No (unknown)";
997 Info("CheckDataType", msg.Data());
998 return;
999 }
1000 TMap *map = coll->Next();
1001 if (!map) {
1002 msg += " using_tags: No (unknown)";
1003 Info("CheckDataType", msg.Data());
1004 return;
1005 }
1006 map = (TMap*)map->GetValue("");
1007 TString file;
1008 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1009 use_tags = file.Contains(".tag");
1010 delete coll;
1011 if (use_tags) msg += " using_tags: Yes";
1012 else msg += " using_tags: No";
1013 Info("CheckDataType", msg.Data());
1014 return;
1015 }
1016 use_tags = slfn.Contains(".tag");
1017 if (slfn.Contains(".root")) msg += " type: root file;";
1018 else msg += " type: unhnown file;";
1019 if (use_tags) msg += " using_tags: Yes";
1020 else msg += " using_tags: No";
1021 Info("CheckDataType", msg.Data());
1022}
1023
1024//______________________________________________________________________________
4e5c5506 1025void AliAnalysisAlien::EnablePackage(const char *package)
1026{
1027// Enables a par file supposed to exist in the current directory.
1028 TString pkg(package);
1029 pkg.ReplaceAll(".par", "");
1030 pkg += ".par";
1031 if (gSystem->AccessPathName(pkg)) {
1032 Error("EnablePackage", "Package %s not found", pkg.Data());
1033 return;
1034 }
1035 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1036 Info("EnablePackage", "AliEn plugin will use .par packages");
1037 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1038 if (!fPackages) {
1039 fPackages = new TObjArray();
1040 fPackages->SetOwner();
1041 }
1042 fPackages->Add(new TObjString(pkg));
1043}
1044
1045//______________________________________________________________________________
d2a409b2 1046const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1047{
1048// Get job status for all jobs with jobid>jobidstart.
1049 static char mstatus[20];
1050 mstatus[0] = '\0';
1051 nrunning = 0;
1052 nwaiting = 0;
1053 nerror = 0;
1054 ndone = 0;
1055 TGridJobStatusList *list = gGrid->Ps("");
1056 if (!list) return mstatus;
1057 Int_t nentries = list->GetSize();
1058 TGridJobStatus *status;
1059 Int_t pid;
1060 for (Int_t ijob=0; ijob<nentries; ijob++) {
1061 status = (TGridJobStatus *)list->At(ijob);
1062 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1063 if (pid<jobidstart) continue;
1064 if (pid == lastid) {
1065 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1066 }
1067 switch (status->GetStatus()) {
1068 case TGridJobStatus::kWAITING:
1069 nwaiting++; break;
1070 case TGridJobStatus::kRUNNING:
1071 nrunning++; break;
1072 case TGridJobStatus::kABORTED:
1073 case TGridJobStatus::kFAIL:
1074 case TGridJobStatus::kUNKNOWN:
1075 nerror++; break;
1076 case TGridJobStatus::kDONE:
1077 ndone++;
1078 }
1079 }
1080 list->Delete();
1081 delete list;
1082 return mstatus;
1083}
1084
1085//______________________________________________________________________________
c57f56b7 1086Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1087{
1088// Returns true if file is a collection. Functionality duplicated from
1089// TAlien::Type() because we don't want to directly depend on TAlien.
1090 if (!gGrid) {
1091 Error("IsCollection", "No connection to grid");
1092 return kFALSE;
1093 }
1094 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1095 if (!res) return kFALSE;
1096 const char* typeStr = res->GetKey(0, "type");
1097 if (!typeStr || !strlen(typeStr)) return kFALSE;
1098 if (!strcmp(typeStr, "collection")) return kTRUE;
1099 delete res;
1100 return kFALSE;
1101}
1102
1103//______________________________________________________________________________
16a4353c 1104void AliAnalysisAlien::Print(Option_t *) const
1105{
1106// Print current plugin settings.
1107 printf("### AliEn analysis plugin current settings ###\n");
a3e84053 1108 printf("= Production mode:______________________________ %d\n", fProductionMode);
16a4353c 1109 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1110 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1111 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1112 if (fUser.Length())
1113 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1114 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1115 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1116 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1117 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1118 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1119 if (fRunNumbers.Length())
1120 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1121 if (fRunRange[0])
1122 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1123 if (!fRunRange[0] && !fRunNumbers.Length()) {
1124 TIter next(fInputFiles);
1125 TObject *obj;
1126 TString list;
1127 while ((obj=next())) list += obj->GetName();
1128 printf("= Input files to be processed: _________________ %s\n", list.Data());
1129 }
1130 if (TestBit(AliAnalysisGrid::kTest))
1131 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1132 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1133 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1134 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1135 printf("=====================================================================\n");
1136 printf("= Job price: ___________________________________ %d\n", fPrice);
1137 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1138 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1139 if (fMaxInitFailed>0)
1140 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1141 if (fMasterResubmitThreshold>0)
1142 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
319593fb 1143 if (fNrunsPerMaster>0)
1144 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
16a4353c 1145 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1146 printf("= Name of the generated execution script: ______ %s\n",fExecutable.Data());
1147 if (fArguments.Length())
1148 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1149 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1150 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1151 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1152 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1153 if (fDatasetName)
1154 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1155 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1156 if (fIncludePath.Data())
1157 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1158 if (fCloseSE.Length())
1159 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1160 if (fFriendChainName.Length())
1161 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1162 if (fPackages) {
1163 TIter next(fPackages);
1164 TObject *obj;
1165 TString list;
1166 while ((obj=next())) list += obj->GetName();
1167 printf("= Par files to be used: ________________________ %s\n", list.Data());
1168 }
1169}
1170
1171//______________________________________________________________________________
c57f56b7 1172void AliAnalysisAlien::SetDefaults()
1173{
1174// Set default values for everything. What cannot be filled will be left empty.
1175 if (fGridJDL) delete fGridJDL;
1176 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1177 fPrice = 1;
1178 fTTL = 30000;
1179 fSplitMaxInputFileNumber = 100;
1180 fMaxInitFailed = 0;
1181 fMasterResubmitThreshold = 0;
bb885a9e 1182 fNtestFiles = 10;
a8739e8a 1183 fRunRange[0] = 0;
1184 fRunRange[1] = 0;
319593fb 1185 fNrunsPerMaster = 1;
16a4353c 1186 fMaxMergeFiles = 100;
c57f56b7 1187 fRunNumbers = "";
1188 fExecutable = "analysis.sh";
1189 fArguments = "";
1190 fAnalysisMacro = "myAnalysis.C";
1191 fAnalysisSource = "";
1192 fAdditionalLibs = "";
1193 fSplitMode = "se";
1194 fAPIVersion = "";
1195 fROOTVersion = "";
1196 fAliROOTVersion = "";
1197 fUser = ""; // Your alien user name
1198 fGridWorkingDir = "";
1199 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1200 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
0df6ccf2 1201 fFriendChainName = "";
c57f56b7 1202 fGridOutputDir = "output";
1203 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1204 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1205 fInputFormat = "xml-single";
1206 fJDLName = "analysis.jdl";
bb885a9e 1207 fMergeExcludes = "";
c57f56b7 1208}
1209
1210//______________________________________________________________________________
1211Bool_t AliAnalysisAlien::MergeOutputs()
1212{
1213// Merge analysis outputs existing in the AliEn space.
1214 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1215 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1216 if (!Connect()) {
1217 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1218 return kFALSE;
1219 }
1220 // Get the output path
d2a409b2 1221 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1222 if (!gGrid->Cd(fGridOutputDir)) {
c57f56b7 1223 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 if (!fOutputFiles.Length()) {
1227 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1228 return kFALSE;
1229 }
1230 TObjArray *list = fOutputFiles.Tokenize(" ");
1231 TIter next(list);
1232 TObjString *str;
1233 TString command;
1234 TString output_file;
16a4353c 1235 TString output_chunk;
1236 TString previous_chunk;
1237 Int_t count_chunk = 0;
1238 Int_t count_zero = fMaxMergeFiles;
c57f56b7 1239 Bool_t merged = kTRUE;
1240 while((str=(TObjString*)next())) {
1241 output_file = str->GetString();
1242 Int_t index = output_file.Index("@");
1243 if (index > 0) output_file.Remove(index);
319593fb 1244 // Skip already merged outputs
1245 if (!gSystem->AccessPathName(output_file)) {
1246 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1247 continue;
1248 }
bb885a9e 1249 if (fMergeExcludes.Length() &&
1250 fMergeExcludes.Contains(output_file.Data())) continue;
16a4353c 1251 // Perform a 'find' command in the output directory, looking for registered outputs
d2a409b2 1252 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
c57f56b7 1253 printf("command: %s\n", command.Data());
1254 TGridResult *res = gGrid->Command(command);
1255 if (!res) continue;
1256 TFileMerger *fm = 0;
1257 TIter nextmap(res);
ab254fd1 1258 TMap *map = 0;
16a4353c 1259 previous_chunk = "";
1260 count_chunk = 0;
319593fb 1261 // Check if there is a merge operation to resume
1262 output_chunk = output_file;
1263 output_chunk.ReplaceAll(".root", "_*.root");
1264 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1265 while (1) {
1266 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1267 if (!map) {
1268 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1269 delete res;
1270 return kFALSE;
1271 }
1272 output_chunk = output_file;
1273 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1274 printf("%s\n", output_chunk.Data());
1275 count_chunk++;
1276 if (gSystem->AccessPathName(output_chunk)) continue;
1277 // Merged file with chunks up to <count_chunk> found
1278 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1279 previous_chunk = output_chunk;
1280 break;
1281 }
1282 }
16a4353c 1283 count_zero = fMaxMergeFiles;
c57f56b7 1284 while ((map=(TMap*)nextmap())) {
16a4353c 1285 // Loop 'find' results and get next LFN
1286 if (count_zero == fMaxMergeFiles) {
1287 // First file in chunk - create file merger and add previous chunk if any.
1288 fm = new TFileMerger(kFALSE);
1289 fm->SetFastMethod(kTRUE);
1290 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1291 output_chunk = output_file;
1292 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1293 }
1294 // If last file found, put merged results in the output file
1295 if (map == res->Last()) output_chunk = output_file;
c57f56b7 1296 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1297 if (!objs || !objs->GetString().Length()) {
16a4353c 1298 // Nothing found - skip this output
c57f56b7 1299 delete res;
16a4353c 1300 delete fm;
1301 break;
1302 }
1303 // Add file to be merged and decrement chunk counter.
1304 fm->AddFile(objs->GetString());
1305 count_zero--;
1306 if (count_zero==0 || map == res->Last()) {
1307 fm->OutputFile(output_chunk);
1308 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1309 // Nothing found - skip this output
1310 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1311 delete res;
1312 delete fm;
1313 break;
1314 }
1315 // Merge the outputs, then go to next chunk
1316 if (!fm->Merge()) {
1317 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1318 delete res;
1319 delete fm;
1320 merged = kFALSE;
1321 break;
1322 } else {
1323 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
319593fb 1324 gSystem->Unlink(previous_chunk);
16a4353c 1325 }
1326 if (map == res->Last()) {
1327 delete res;
1328 delete fm;
1329 break;
1330 }
1331 count_chunk++;
1332 count_zero = fMaxMergeFiles;
1333 previous_chunk = output_chunk;
c57f56b7 1334 }
c57f56b7 1335 }
c57f56b7 1336 }
1337 if (!merged) {
1338 Error("MergeOutputs", "Terminate() will NOT be executed");
1339 }
1340 return merged;
1341}
1342
1343//______________________________________________________________________________
bb885a9e 1344void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1345{
1346// Use the output files connected to output containers from the analysis manager
1347// rather than the files defined by SetOutputFiles
1348 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1349 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1350 analysis manager");
1351 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1352}
1353
1354//______________________________________________________________________________
5513444a 1355Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
c57f56b7 1356{
1357// Start remote grid analysis.
1358
43da816a 1359 // Check if output files have to be taken from the analysis manager
1360 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1361 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1362 if (!mgr || !mgr->IsInitialized()) {
1363 Error("StartAnalysis", "You need an initialized analysis manager for this");
5513444a 1364 return kFALSE;
43da816a 1365 }
1366 fOutputFiles = "";
1367 TIter next(mgr->GetOutputs());
1368 AliAnalysisDataContainer *output;
1369 while ((output=(AliAnalysisDataContainer*)next())) {
1370 const char *filename = output->GetFileName();
1371 if (!(strcmp(filename, "default"))) {
1372 if (!mgr->GetOutputEventHandler()) continue;
1373 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1374 }
1375 if (fOutputFiles.Length()) fOutputFiles += " ";
1376 fOutputFiles += filename;
1377 }
c07b9ce2 1378 // Add extra files registered to the analysis manager
1379 if (mgr->GetExtraFiles().Length()) {
1380 if (fOutputFiles.Length()) fOutputFiles += " ";
1381 fOutputFiles += mgr->GetExtraFiles();
1382 }
43da816a 1383 }
f7b1cbc2 1384// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
c57f56b7 1385 if (TestBit(AliAnalysisGrid::kOffline)) {
1386 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1387 \n there nor any job run. You can revise the JDL and analysis \
1388 \n macro then run the same in \"submit\" mode.");
1389 } else if (TestBit(AliAnalysisGrid::kTest)) {
1390 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1391 \n dataset.");
1392 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1393 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1394 \n space and job submitted.");
1395 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1396 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
5513444a 1397 return kTRUE;
c57f56b7 1398 } else {
1399 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1400 }
1401
1402 if (!Connect()) {
1403 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
5513444a 1404 return kFALSE;
16a4353c 1405 }
1406 Print();
c57f56b7 1407 if (!CheckInputData()) {
1408 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
5513444a 1409 return kFALSE;
c57f56b7 1410 }
1411 CreateDataset(fDataPattern);
1412 WriteAnalysisFile();
1413 WriteAnalysisMacro();
1414 WriteExecutable();
1415 WriteValidationScript();
5513444a 1416 if (!CreateJDL()) return kFALSE;
1417 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
c57f56b7 1418 if (TestBit(AliAnalysisGrid::kTest)) {
1419 // Locally testing the analysis
1420 Info("StartAnalysis", "\n_______________________________________________________________________ \
1421 \n Running analysis script in a daughter shell as on a worker node \
1422 \n_______________________________________________________________________");
1423 TObjArray *list = fOutputFiles.Tokenize(" ");
1424 TIter next(list);
1425 TObjString *str;
1426 TString output_file;
1427 while((str=(TObjString*)next())) {
1428 output_file = str->GetString();
1429 Int_t index = output_file.Index("@");
1430 if (index > 0) output_file.Remove(index);
43da816a 1431 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
c57f56b7 1432 }
1433 delete list;
1434 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1435 gSystem->Exec("bash validate.sh");
1436// gSystem->Exec("cat stdout");
5513444a 1437 return kFALSE;
c57f56b7 1438 }
5513444a 1439 // Check if submitting is managed by LPM manager
a3e84053 1440 if (fProductionMode) {
5513444a 1441 TString prodfile = fJDLName;
1442 prodfile.ReplaceAll(".jdl", ".prod");
1443 WriteProductionFile(prodfile);
1444 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1445 return kFALSE;
1446 }
a8739e8a 1447 // Submit AliEn job(s)
d2a409b2 1448 gGrid->Cd(fGridOutputDir);
a8739e8a 1449 TGridResult *res;
c57f56b7 1450 TString jobID = "";
d2a409b2 1451 if (!fRunNumbers.Length() && !fRunRange[0]) {
dd74a515 1452 // Submit a given xml or a set of runs
a8739e8a 1453 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1454 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1455 if (res) {
1456 const char *cjobId = res->GetKey(0,"jobId");
1457 if (!cjobId) {
1458 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
5513444a 1459 return kFALSE;
a8739e8a 1460 } else {
1461 Info("StartAnalysis", "\n_______________________________________________________________________ \
1462 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1463 \n_______________________________________________________________________",
1464 fJDLName.Data(), cjobId);
1465 jobID = cjobId;
1466 }
1467 delete res;
1468 }
1469 } else {
d2a409b2 1470 // Submit for a range of enumeration of runs.
1471 Submit();
c57f56b7 1472 }
a8739e8a 1473
c57f56b7 1474 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1475 \n You may exit at any time and terminate the job later using the option <terminate> \
1476 \n ##################################################################################", jobID.Data());
bb885a9e 1477 gSystem->Exec("aliensh");
5513444a 1478 return kTRUE;
c57f56b7 1479}
1480
1481//______________________________________________________________________________
d2a409b2 1482void AliAnalysisAlien::Submit()
1483{
1484// Submit all master jobs.
1485 Int_t nmasterjobs = fInputFiles->GetEntries();
1486 Long_t tshoot = gSystem->Now();
1487 if (!fNsubmitted) SubmitNext();
1488 while (fNsubmitted < nmasterjobs) {
1489 Long_t now = gSystem->Now();
1490 if ((now-tshoot)>30000) {
1491 tshoot = now;
1492 SubmitNext();
1493 }
1494 }
1495}
1496
1497//______________________________________________________________________________
1498void AliAnalysisAlien::SubmitNext()
1499{
1500// Submit next bunch of master jobs if the queue is free.
1501 static Bool_t iscalled = kFALSE;
1502 static Int_t firstmaster = 0;
1503 static Int_t lastmaster = 0;
1504 static Int_t npermaster = 0;
1505 if (iscalled) return;
1506 iscalled = kTRUE;
1507 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1508 Int_t ntosubmit = 0;
1509 TGridResult *res;
1510 TString jobID = "";
1511 if (!fNsubmitted) ntosubmit = 1;
1512 else {
1513 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
1514 printf("=== master %d: %s\n", lastmaster, status.Data());
1515 // If last master not split, just return
1516 if (status != "SPLIT") {iscalled = kFALSE; return;}
1517 // No more than 100 waiting jobs
1518 if (nwaiting>100) {iscalled = kFALSE; return;}
1519 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1520 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
1521 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
1522 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1523 }
1524 Int_t nmasterjobs = fInputFiles->GetEntries();
1525 for (Int_t i=0; i<ntosubmit; i++) {
1526 // Submit for a range of enumeration of runs.
1527 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1528 TString query;
1529 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
1530 printf("********* %s\n",query.Data());
1531 res = gGrid->Command(query);
1532 if (res) {
98ca124f 1533 TString cjobId1 = res->GetKey(0,"jobId");
1534 if (!cjobId1.Length()) {
d2a409b2 1535 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1536 iscalled = kFALSE;
1537 return;
1538 } else {
1539 Info("StartAnalysis", "\n_______________________________________________________________________ \
1540 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1541 \n_______________________________________________________________________",
98ca124f 1542 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
d2a409b2 1543 jobID += cjobId1;
1544 jobID += " ";
98ca124f 1545 lastmaster = cjobId1.Atoi();
d2a409b2 1546 if (!firstmaster) firstmaster = lastmaster;
1547 fNsubmitted++;
1548 }
1549 delete res;
1550 }
1551 }
1552 iscalled = kFALSE;
1553}
1554
1555//______________________________________________________________________________
c57f56b7 1556void AliAnalysisAlien::WriteAnalysisFile()
1557{
1558// Write current analysis manager into the file analysis.root
1559 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1560 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1561 if (!mgr || !mgr->IsInitialized()) {
1562 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1563 return;
1564 }
1565 // Check analysis type
1566 TObject *handler;
1567 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1568 handler = (TObject*)mgr->GetInputEventHandler();
1569 if (handler) {
1570 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1571 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1572 }
1573 TDirectory *cdir = gDirectory;
1574 TFile *file = TFile::Open("analysis.root", "RECREATE");
1575 if (file) {
1576 mgr->Write();
1577 delete file;
1578 }
1579 if (cdir) cdir->cd();
1580 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1581 }
1582 Bool_t copy = kTRUE;
1583 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1584 if (copy) {
1585 CdWork();
1586 TString workdir = gGrid->GetHomeDirectory();
1587 workdir += fGridWorkingDir;
1588 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1589 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1590 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1591 }
1592}
1593
1594//______________________________________________________________________________
1595void AliAnalysisAlien::WriteAnalysisMacro()
1596{
1597// Write the analysis macro that will steer the analysis in grid mode.
1598 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1599 ofstream out;
1600 out.open(fAnalysisMacro.Data(), ios::out);
1601 if (!out.good()) {
1602 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1603 return;
1604 }
1605 TString func = fAnalysisMacro;
1606 TString type = "ESD";
1607 TString comment = "// Analysis using ";
1608 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1609 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1610 type = "AOD";
1611 comment += "AOD";
1612 }
0df6ccf2 1613 if (type!="AOD" && fFriendChainName!="") {
1614 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1615 return;
1616 }
c57f56b7 1617 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1618 else comment += " data";
1619 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1620 func.ReplaceAll(".C", "");
1621 out << "void " << func.Data() << "()" << endl;
1622 out << "{" << endl;
1623 out << comment.Data() << endl;
1624 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
f7498086 1625 out << " TStopwatch timer;" << endl;
1626 out << " timer.Start();" << endl << endl;
c57f56b7 1627 out << "// load base root libraries" << endl;
1628 out << " gSystem->Load(\"libTree\");" << endl;
1629 out << " gSystem->Load(\"libGeom\");" << endl;
1630 out << " gSystem->Load(\"libVMC\");" << endl;
1631 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
57377eb5 1632 out << "// Load analysis framework libraries" << endl;
4e5c5506 1633 if (!fPackages) {
4e5c5506 1634 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1635 out << " gSystem->Load(\"libESD\");" << endl;
1636 out << " gSystem->Load(\"libAOD\");" << endl;
1637 out << " gSystem->Load(\"libANALYSIS\");" << endl;
57377eb5 1638 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1639 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4e5c5506 1640 } else {
4e5c5506 1641 TIter next(fPackages);
1642 TObject *obj;
57377eb5 1643 TString pkgname;
1644 Bool_t hasSTEERBase = kFALSE;
1645 Bool_t hasESD = kFALSE;
1646 Bool_t hasAOD = kFALSE;
1647 Bool_t hasANALYSIS = kFALSE;
1648 Bool_t hasANALYSISalice = kFALSE;
1649 Bool_t hasCORRFW = kFALSE;
1650 while ((obj=next())) {
1651 pkgname = obj->GetName();
4478e6f1 1652 if (pkgname == "STEERBase" ||
1653 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1654 if (pkgname == "ESD" ||
1655 pkgname == "ESD.par") hasESD = kTRUE;
1656 if (pkgname == "AOD" ||
1657 pkgname == "AOD.par") hasAOD = kTRUE;
1658 if (pkgname == "ANALYSIS" ||
1659 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1660 if (pkgname == "ANALYSISalice" ||
1661 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1662 if (pkgname == "CORRFW" ||
1663 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
57377eb5 1664 }
1665 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1666 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1667 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1668 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1669 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1670 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1671 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1672 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1673 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1674 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1675 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1676 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1677 out << "// Compile other par packages" << endl;
1678 next.Reset();
fcc9bb6f 1679 while ((obj=next())) {
1680 pkgname = obj->GetName();
4478e6f1 1681 if (pkgname == "STEERBase" ||
1682 pkgname == "STEERBase.par" ||
1683 pkgname == "ESD" ||
1684 pkgname == "ESD.par" ||
1685 pkgname == "AOD" ||
1686 pkgname == "AOD.par" ||
1687 pkgname == "ANALYSIS" ||
1688 pkgname == "ANALYSIS.par" ||
1689 pkgname == "ANALYSISalice" ||
1690 pkgname == "ANALYSISalice.par" ||
1691 pkgname == "CORRFW" ||
1692 pkgname == "CORRFW.par") continue;
4e5c5506 1693 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
fcc9bb6f 1694 }
4e5c5506 1695 }
43da816a 1696 out << "// include path" << endl;
1697 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1698 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
6da75e0b 1699 if (fAdditionalLibs.Length()) {
1700 out << "// Add aditional AliRoot libraries" << endl;
1701 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1702 TIter next(list);
1703 TObjString *str;
1704 while((str=(TObjString*)next())) {
1705 if (str->GetString().Contains(".so"))
1706 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1707 }
1708 if (list) delete list;
1709 }
1710 out << endl;
c57f56b7 1711 out << "// analysis source to be compiled at runtime (if any)" << endl;
1712 if (fAnalysisSource.Length()) {
1713 TObjArray *list = fAnalysisSource.Tokenize(" ");
1714 TIter next(list);
1715 TObjString *str;
1716 while((str=(TObjString*)next())) {
1717 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1718 }
1719 if (list) delete list;
1720 }
1721 out << endl;
1722 out << "// connect to AliEn and make the chain" << endl;
1723 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1724 if (IsUsingTags()) {
1725 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1726 } else {
1727 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1728 }
1729 out << "// read the analysis manager from file" << endl;
1730 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1731 out << " if (!file) return;" << endl;
1732 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1733 out << " AliAnalysisManager *mgr = 0;" << endl;
1734 out << " TKey *key;" << endl;
1735 out << " while ((key=(TKey*)nextkey())) {" << endl;
1736 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1737 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1738 out << " };" << endl;
1739 out << " if (!mgr) {" << endl;
1740 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1741 out << " return;" << endl;
1742 out << " }" << endl << endl;
1743 out << " mgr->PrintStatus();" << endl;
1744 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
f7498086 1745 out << " timer.Stop();" << endl;
1746 out << " timer.Print();" << endl;
c57f56b7 1747 out << "}" << endl << endl;
1748 if (IsUsingTags()) {
1749 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1750 out << "{" << endl;
1751 out << "// Create a chain using tags from the xml file." << endl;
1752 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1753 out << " if (!coll) {" << endl;
1754 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1755 out << " return NULL;" << endl;
1756 out << " }" << endl;
1757 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1758 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1759 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1760 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1761 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1762 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1763 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1764 out << " // Check if the cuts configuration file was provided" << endl;
1765 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1766 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1767 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1768 out << " }" << endl;
0df6ccf2 1769 if (fFriendChainName=="") {
1770 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1771 } else {
1772 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1773 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1774 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1775 }
c57f56b7 1776 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1777 out << " chain->ls();" << endl;
1778 out << " return chain;" << endl;
fcc9bb6f 1779 out << "}" << endl << endl;
c57f56b7 1780 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1781 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1782 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1783 msg += " AliLHCTagCuts *lhcCuts,\n";
1784 msg += " AliDetectorTagCuts *detCuts,\n";
1785 msg += " AliEventTagCuts *evCuts)";
1786 Info("WriteAnalysisMacro", msg.Data());
1787 }
0df6ccf2 1788 }
1789 if (!IsUsingTags() || fFriendChainName!="") {
fcc9bb6f 1790 out <<"//________________________________________________________________________________" << endl;
c57f56b7 1791 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1792 out << "{" << endl;
1793 out << "// Create a chain using url's from xml file" << endl;
1794 out << " TString treename = type;" << endl;
1795 out << " treename.ToLower();" << endl;
1796 out << " treename += \"Tree\";" << endl;
e02fee64 1797 out << " printf(\"***************************************\\n\");" << endl;
1798 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1799 out << " printf(\"***************************************\\n\");" << endl;
c57f56b7 1800 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1801 out << " if (!coll) {" << endl;
1802 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1803 out << " return NULL;" << endl;
1804 out << " }" << endl;
1805 out << " TChain *chain = new TChain(treename);" << endl;
0df6ccf2 1806 if(fFriendChainName!="") {
1807 out << " TChain *chainFriend = new TChain(treename);" << endl;
1808 }
c57f56b7 1809 out << " coll->Reset();" << endl;
0df6ccf2 1810 out << " while (coll->Next()) {" << endl;
1811 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1812 if(fFriendChainName!="") {
1813 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1814 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1815 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1816 out << " chainFriend->Add(fileFriend.Data());" << endl;
1817 }
1818 out << " }" << endl;
c57f56b7 1819 out << " if (!chain->GetNtrees()) {" << endl;
1820 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1821 out << " return NULL;" << endl;
1822 out << " }" << endl;
0df6ccf2 1823 if(fFriendChainName!="") {
1824 out << " chain->AddFriend(chainFriend);" << endl;
1825 }
c57f56b7 1826 out << " return chain;" << endl;
fcc9bb6f 1827 out << "}" << endl << endl;
c57f56b7 1828 }
4e5c5506 1829 if (fPackages) {
fcc9bb6f 1830 out <<"//________________________________________________________________________________" << endl;
4e5c5506 1831 out << "Bool_t SetupPar(const char *package) {" << endl;
1832 out << "// Compile the package and set it up." << endl;
1833 out << " TString pkgdir = package;" << endl;
1834 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
fcc9bb6f 1835 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4e5c5506 1836 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1837 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1838 out << " // Check for BUILD.sh and execute" << endl;
1839 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1840 out << " printf(\"*******************************\\n\");" << endl;
1841 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1842 out << " printf(\"*******************************\\n\");" << endl;
1843 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
fcc9bb6f 1844 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4e5c5506 1845 out << " gSystem->ChangeDirectory(cdir);" << endl;
1846 out << " return kFALSE;" << endl;
1847 out << " }" << endl;
1848 out << " } else {" << endl;
fcc9bb6f 1849 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4e5c5506 1850 out << " gSystem->ChangeDirectory(cdir);" << endl;
1851 out << " return kFALSE;" << endl;
1852 out << " }" << endl;
1853 out << " // Check for SETUP.C and execute" << endl;
1854 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1855 out << " printf(\"*******************************\\n\");" << endl;
1856 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1857 out << " printf(\"*******************************\\n\");" << endl;
1858 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1859 out << " } else {" << endl;
fcc9bb6f 1860 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4e5c5506 1861 out << " gSystem->ChangeDirectory(cdir);" << endl;
1862 out << " return kFALSE;" << endl;
1863 out << " }" << endl;
1864 out << " // Restore original workdir" << endl;
1865 out << " gSystem->ChangeDirectory(cdir);" << endl;
1866 out << " return kTRUE;" << endl;
1867 out << "}" << endl;
1868 }
c57f56b7 1869 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1870 }
1871 Bool_t copy = kTRUE;
1872 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1873 if (copy) {
1874 CdWork();
1875 TString workdir = gGrid->GetHomeDirectory();
1876 workdir += fGridWorkingDir;
1877 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1878 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1879 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1880 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1881 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1882 }
1883 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1884 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1885 }
1886}
1887
1888//______________________________________________________________________________
1889void AliAnalysisAlien::WriteExecutable()
1890{
1891// Generate the alien executable script.
1892 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1893 ofstream out;
1894 out.open(fExecutable.Data(), ios::out);
1895 if (out.bad()) {
5513444a 1896 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
c57f56b7 1897 return;
1898 }
1899 out << "#!/bin/bash" << endl;
1900 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1901 out << "echo \"=========================================\"" << endl;
1902 out << "echo \"############## PATH : ##############\"" << endl;
1903 out << "echo $PATH" << endl;
1904 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1905 out << "echo $LD_LIBRARY_PATH" << endl;
1906 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1907 out << "echo $ROOTSYS" << endl;
1908 out << "echo \"############## which root : ##############\"" << endl;
1909 out << "which root" << endl;
1910 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1911 out << "echo $ALICE_ROOT" << endl;
1912 out << "echo \"############## which aliroot : ##############\"" << endl;
1913 out << "which aliroot" << endl;
1914 out << "echo \"=========================================\"" << endl << endl;
1915// if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1916 out << "root -b -q ";
1917 out << fAnalysisMacro.Data() << endl << endl;
1918 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1919 }
1920 Bool_t copy = kTRUE;
1921 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1922 if (copy) {
1923 CdWork();
1924 TString workdir = gGrid->GetHomeDirectory();
1925 workdir += fGridWorkingDir;
1926 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1927 if (FileExists(executable)) gGrid->Rm(executable);
1928 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1929 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1930 }
1931}
1932
1933//______________________________________________________________________________
5513444a 1934void AliAnalysisAlien::WriteProductionFile(const char *filename) const
1935{
1936// Write the production file to be submitted by LPM manager. The format is:
f5e8c702 1937// First line: full_path_to_jdl estimated_no_subjobs_per_master
5513444a 1938// Next lines: full_path_to_dataset XXX (XXX is a string)
1939// To submit, one has to: submit jdl XXX for all lines
1940 ofstream out;
1941 out.open(filename, ios::out);
1942 if (out.bad()) {
1943 Error("WriteProductionFile", "Bad file name: %s", filename);
1944 return;
1945 }
1946 TString workdir = gGrid->GetHomeDirectory();
1947 workdir += fGridWorkingDir;
f5e8c702 1948 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5513444a 1949 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
f5e8c702 1950 out << locjdl << " " << njobspermaster << endl;
5513444a 1951 Int_t nmasterjobs = fInputFiles->GetEntries();
1952 for (Int_t i=0; i<nmasterjobs; i++) {
b5fe9cba 1953 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5513444a 1954 }
1955 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
1956 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
1957}
1958
1959//______________________________________________________________________________
c57f56b7 1960void AliAnalysisAlien::WriteValidationScript()
1961{
1962// Generate the alien validation script.
1963 // Generate the validation script
1964 TObjString *os;
1965 if (!Connect()) {
1966 Error("WriteValidationScript", "Alien connection required");
1967 return;
1968 }
1969 TString out_stream = "";
1970 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1971 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1972 ofstream out;
1973 out.open("validate.sh", ios::out);
1974 out << "#!/bin/bash" << endl;
1975 out << "##################################################" << endl;
1976 out << "validateout=`dirname $0`" << endl;
1977 out << "validatetime=`date`" << endl;
1978 out << "validated=\"0\";" << endl;
1979 out << "error=0" << endl;
1980 out << "if [ -z $validateout ]" << endl;
1981 out << "then" << endl;
1982 out << " validateout=\".\"" << endl;
1983 out << "fi" << endl << endl;
1984 out << "cd $validateout;" << endl;
1985 out << "validateworkdir=`pwd`;" << endl << endl;
1986 out << "echo \"*******************************************************\"" << out_stream << endl;
1987 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1988 out << "" << endl;
1989 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1990 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1991 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1992 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1993 out << "ls -la ./" << out_stream << endl;
1994 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1995 out << "##################################################" << endl;
ebec370a 1996
1997 out << "" << endl;
1998 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
1999 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2000 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2001 out << "" << endl;
2002
2003 out << "if [ ! -f stderr ] ; then" << endl;
2004 out << " error=1" << endl;
2005 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2006 out << " echo \"Error = $error\" " << out_stream << endl;
2007 out << "fi" << endl;
2008
2009 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2010 out << " error=1" << endl;
2011 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2012 out << " echo \"$parArch\" " << out_stream << endl;
2013 out << " echo \"Error = $error\" " << out_stream << endl;
2014 out << "fi" << endl;
2015
2016 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2017 out << " error=1" << endl;
2018 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2019 out << " echo \"$segViol\" " << out_stream << endl;
2020 out << " echo \"Error = $error\" " << out_stream << endl;
2021 out << "fi" << endl;
2022
2023 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2024 out << " error=1" << endl;
2025 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2026 out << " echo \"$segFault\" " << out_stream << endl;
2027 out << " echo \"Error = $error\" " << out_stream << endl;
2028 out << "fi" << endl;
2029
2030 // Part dedicated to the specific analyses running into the train
2031
c57f56b7 2032 TObjArray *arr = fOutputFiles.Tokenize(" ");
2033 TIter next1(arr);
2034 TString output_file;
2035 while ((os=(TObjString*)next1())) {
2036 output_file = os->GetString();
2037 Int_t index = output_file.Index("@");
2038 if (index > 0) output_file.Remove(index);
2039 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2040 out << " error=1" << endl;
2041 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2042 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2043 out << "fi" << endl;
2044 }
2045 delete arr;
2046 out << "if [ $error = 0 ] ; then" << endl;
2047 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2048 out << "fi" << endl;
2049
2050 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2051 out << "echo \"*******************************************************\"" << out_stream << endl;
2052 out << "cd -" << endl;
2053 out << "exit $error" << endl;
2054 }
2055 Bool_t copy = kTRUE;
2056 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2057 if (copy) {
2058 CdWork();
2059 TString workdir = gGrid->GetHomeDirectory();
2060 workdir += fGridWorkingDir;
2061 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
2062 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
2063 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
2064 }
2065}