]> git.uio.no Git - u/mrichter/AliRoot.git/blame_incremental - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Read histos from common output file (Chiara B)
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
... / ...
CommitLineData
1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TROOT.h"
25#include "TSystem.h"
26#include "TFile.h"
27#include "TObjString.h"
28#include "TObjArray.h"
29#include "TGrid.h"
30#include "TGridResult.h"
31#include "TGridCollection.h"
32#include "TGridJDL.h"
33#include "TGridJobStatusList.h"
34#include "TGridJobStatus.h"
35#include "TFileMerger.h"
36#include "AliAnalysisManager.h"
37#include "AliVEventHandler.h"
38#include "AliAnalysisDataContainer.h"
39#include "AliAnalysisAlien.h"
40
41ClassImp(AliAnalysisAlien)
42
43//______________________________________________________________________________
44AliAnalysisAlien::AliAnalysisAlien()
45 :AliAnalysisGrid(),
46 fGridJDL(NULL),
47 fPrice(0),
48 fTTL(0),
49 fSplitMaxInputFileNumber(0),
50 fMaxInitFailed(0),
51 fMasterResubmitThreshold(0),
52 fNtestFiles(0),
53 fNrunsPerMaster(0),
54 fMaxMergeFiles(0),
55 fNsubmitted(0),
56 fProductionMode(0),
57 fOutputToRunNo(0),
58 fRunNumbers(),
59 fExecutable(),
60 fExecutableCommand(),
61 fArguments(),
62 fExecutableArgs(),
63 fAnalysisMacro(),
64 fAnalysisSource(),
65 fAdditionalLibs(),
66 fSplitMode(),
67 fAPIVersion(),
68 fROOTVersion(),
69 fAliROOTVersion(),
70 fExternalPackages(),
71 fUser(),
72 fGridWorkingDir(),
73 fGridDataDir(),
74 fDataPattern(),
75 fGridOutputDir(),
76 fOutputArchive(),
77 fOutputFiles(),
78 fInputFormat(),
79 fDatasetName(),
80 fJDLName(),
81 fMergeExcludes(),
82 fIncludePath(),
83 fCloseSE(),
84 fFriendChainName(),
85 fJobTag(),
86 fOutputSingle(),
87 fRunPrefix(),
88 fInputFiles(0),
89 fPackages(0)
90{
91// Dummy ctor.
92 SetDefaults();
93}
94
95//______________________________________________________________________________
96AliAnalysisAlien::AliAnalysisAlien(const char *name)
97 :AliAnalysisGrid(name),
98 fGridJDL(NULL),
99 fPrice(0),
100 fTTL(0),
101 fSplitMaxInputFileNumber(0),
102 fMaxInitFailed(0),
103 fMasterResubmitThreshold(0),
104 fNtestFiles(0),
105 fNrunsPerMaster(0),
106 fMaxMergeFiles(0),
107 fNsubmitted(0),
108 fProductionMode(0),
109 fOutputToRunNo(0),
110 fRunNumbers(),
111 fExecutable(),
112 fExecutableCommand(),
113 fArguments(),
114 fExecutableArgs(),
115 fAnalysisMacro(),
116 fAnalysisSource(),
117 fAdditionalLibs(),
118 fSplitMode(),
119 fAPIVersion(),
120 fROOTVersion(),
121 fAliROOTVersion(),
122 fExternalPackages(),
123 fUser(),
124 fGridWorkingDir(),
125 fGridDataDir(),
126 fDataPattern(),
127 fGridOutputDir(),
128 fOutputArchive(),
129 fOutputFiles(),
130 fInputFormat(),
131 fDatasetName(),
132 fJDLName(),
133 fMergeExcludes(),
134 fIncludePath(),
135 fCloseSE(),
136 fFriendChainName(),
137 fJobTag(),
138 fOutputSingle(),
139 fRunPrefix(),
140 fInputFiles(0),
141 fPackages(0)
142{
143// Default ctor.
144 SetDefaults();
145}
146
147//______________________________________________________________________________
148AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
149 :AliAnalysisGrid(other),
150 fGridJDL(NULL),
151 fPrice(other.fPrice),
152 fTTL(other.fTTL),
153 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
154 fMaxInitFailed(other.fMaxInitFailed),
155 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
156 fNtestFiles(other.fNtestFiles),
157 fNrunsPerMaster(other.fNrunsPerMaster),
158 fMaxMergeFiles(other.fMaxMergeFiles),
159 fNsubmitted(other.fNsubmitted),
160 fProductionMode(other.fProductionMode),
161 fOutputToRunNo(other.fOutputToRunNo),
162 fRunNumbers(other.fRunNumbers),
163 fExecutable(other.fExecutable),
164 fExecutableCommand(other.fExecutableCommand),
165 fArguments(other.fArguments),
166 fExecutableArgs(other.fExecutableArgs),
167 fAnalysisMacro(other.fAnalysisMacro),
168 fAnalysisSource(other.fAnalysisSource),
169 fAdditionalLibs(other.fAdditionalLibs),
170 fSplitMode(other.fSplitMode),
171 fAPIVersion(other.fAPIVersion),
172 fROOTVersion(other.fROOTVersion),
173 fAliROOTVersion(other.fAliROOTVersion),
174 fExternalPackages(other.fExternalPackages),
175 fUser(other.fUser),
176 fGridWorkingDir(other.fGridWorkingDir),
177 fGridDataDir(other.fGridDataDir),
178 fDataPattern(other.fDataPattern),
179 fGridOutputDir(other.fGridOutputDir),
180 fOutputArchive(other.fOutputArchive),
181 fOutputFiles(other.fOutputFiles),
182 fInputFormat(other.fInputFormat),
183 fDatasetName(other.fDatasetName),
184 fJDLName(other.fJDLName),
185 fMergeExcludes(other.fMergeExcludes),
186 fIncludePath(other.fIncludePath),
187 fCloseSE(other.fCloseSE),
188 fFriendChainName(other.fFriendChainName),
189 fJobTag(other.fJobTag),
190 fOutputSingle(other.fOutputSingle),
191 fRunPrefix(other.fRunPrefix),
192 fInputFiles(0),
193 fPackages(0)
194{
195// Copy ctor.
196 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
197 fRunRange[0] = other.fRunRange[0];
198 fRunRange[1] = other.fRunRange[1];
199 if (other.fInputFiles) {
200 fInputFiles = new TObjArray();
201 TIter next(other.fInputFiles);
202 TObject *obj;
203 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
204 fInputFiles->SetOwner();
205 }
206 if (other.fPackages) {
207 fPackages = new TObjArray();
208 TIter next(other.fPackages);
209 TObject *obj;
210 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
211 fPackages->SetOwner();
212 }
213}
214
215//______________________________________________________________________________
216AliAnalysisAlien::~AliAnalysisAlien()
217{
218// Destructor.
219 if (fGridJDL) delete fGridJDL;
220 if (fInputFiles) delete fInputFiles;
221 if (fPackages) delete fPackages;
222}
223
224//______________________________________________________________________________
225AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
226{
227// Assignment.
228 if (this != &other) {
229 AliAnalysisGrid::operator=(other);
230 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
231 fPrice = other.fPrice;
232 fTTL = other.fTTL;
233 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
234 fMaxInitFailed = other.fMaxInitFailed;
235 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
236 fNtestFiles = other.fNtestFiles;
237 fNrunsPerMaster = other.fNrunsPerMaster;
238 fMaxMergeFiles = other.fMaxMergeFiles;
239 fNsubmitted = other.fNsubmitted;
240 fProductionMode = other.fProductionMode;
241 fOutputToRunNo = other.fOutputToRunNo;
242 fRunNumbers = other.fRunNumbers;
243 fExecutable = other.fExecutable;
244 fExecutableCommand = other.fExecutableCommand;
245 fArguments = other.fArguments;
246 fExecutableArgs = other.fExecutableArgs;
247 fAnalysisMacro = other.fAnalysisMacro;
248 fAnalysisSource = other.fAnalysisSource;
249 fAdditionalLibs = other.fAdditionalLibs;
250 fSplitMode = other.fSplitMode;
251 fAPIVersion = other.fAPIVersion;
252 fROOTVersion = other.fROOTVersion;
253 fAliROOTVersion = other.fAliROOTVersion;
254 fExternalPackages = other.fExternalPackages;
255 fUser = other.fUser;
256 fGridWorkingDir = other.fGridWorkingDir;
257 fGridDataDir = other.fGridDataDir;
258 fDataPattern = other.fDataPattern;
259 fGridOutputDir = other.fGridOutputDir;
260 fOutputArchive = other.fOutputArchive;
261 fOutputFiles = other.fOutputFiles;
262 fInputFormat = other.fInputFormat;
263 fDatasetName = other.fDatasetName;
264 fJDLName = other.fJDLName;
265 fMergeExcludes = other.fMergeExcludes;
266 fIncludePath = other.fIncludePath;
267 fCloseSE = other.fCloseSE;
268 fFriendChainName = other.fFriendChainName;
269 fJobTag = other.fJobTag;
270 fOutputSingle = other.fOutputSingle;
271 fRunPrefix = other.fRunPrefix;
272 if (other.fInputFiles) {
273 fInputFiles = new TObjArray();
274 TIter next(other.fInputFiles);
275 TObject *obj;
276 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
277 fInputFiles->SetOwner();
278 }
279 if (other.fPackages) {
280 fPackages = new TObjArray();
281 TIter next(other.fPackages);
282 TObject *obj;
283 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
284 fPackages->SetOwner();
285 }
286 }
287 return *this;
288}
289
290//______________________________________________________________________________
291void AliAnalysisAlien::AddIncludePath(const char *path)
292{
293// Add include path in the remote analysis macro.
294 TString p(path);
295 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
296 else fIncludePath += Form("-I%s ", path);
297}
298
299//______________________________________________________________________________
300void AliAnalysisAlien::AddRunNumber(Int_t run)
301{
302// Add a run number to the list of runs to be processed.
303 if (fRunNumbers.Length()) fRunNumbers += " ";
304 fRunNumbers += Form("%s%d", fRunPrefix.Data(), run);
305}
306
307//______________________________________________________________________________
308void AliAnalysisAlien::AddRunNumber(const char* run)
309{
310// Add a run number to the list of runs to be processed.
311 if (fRunNumbers.Length()) fRunNumbers += " ";
312 fRunNumbers += run;
313}
314
315//______________________________________________________________________________
316void AliAnalysisAlien::AddDataFile(const char *lfn)
317{
318// Adds a data file to the input to be analysed. The file should be a valid LFN
319// or point to an existing file in the alien workdir.
320 if (!fInputFiles) fInputFiles = new TObjArray();
321 fInputFiles->Add(new TObjString(lfn));
322}
323
324//______________________________________________________________________________
325void AliAnalysisAlien::AddExternalPackage(const char *package)
326{
327// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
328 if (fExternalPackages) fExternalPackages += " ";
329 fExternalPackages += package;
330}
331
332//______________________________________________________________________________
333Bool_t AliAnalysisAlien::Connect()
334{
335// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
336 if (gGrid && gGrid->IsConnected()) return kTRUE;
337 if (!gSystem->Getenv("alien_API_USER")) {
338 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
339 gSystem->Getenv("UID"));
340 return kFALSE;
341 }
342 if (!gGrid) {
343 Info("Connect", "Trying to connect to AliEn ...");
344 TGrid::Connect("alien://");
345 }
346 if (!gGrid || !gGrid->IsConnected()) {
347 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
348 return kFALSE;
349 }
350 fUser = gGrid->GetUser();
351 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
352 return kTRUE;
353}
354
355//______________________________________________________________________________
356void AliAnalysisAlien::CdWork()
357{
358// Check validity of alien workspace. Create directory if possible.
359 if (!Connect()) {
360 Error("CdWork", "Alien connection required");
361 return;
362 }
363 TString homedir = gGrid->GetHomeDirectory();
364 TString workdir = homedir + fGridWorkingDir;
365 if (DirectoryExists(workdir)) {
366 gGrid->Cd(workdir);
367 return;
368 }
369 // Work directory not existing - create it
370 gGrid->Cd(homedir);
371 if (gGrid->Mkdir(workdir)) {
372 gGrid->Cd(fGridWorkingDir);
373 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
374 } else {
375 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
376 workdir.Data(), homedir.Data());
377 fGridWorkingDir = "";
378 }
379}
380
381//______________________________________________________________________________
382Bool_t AliAnalysisAlien::CheckInputData()
383{
384// Check validity of input data. If necessary, create xml files.
385 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
386 if (!fGridDataDir.Length()) {
387 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
388 return kFALSE;
389 }
390 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
391 return kTRUE;
392 }
393 // Process declared files
394 Bool_t is_collection = kFALSE;
395 Bool_t is_xml = kFALSE;
396 Bool_t use_tags = kFALSE;
397 Bool_t checked = kFALSE;
398 CdWork();
399 TString file;
400 TString workdir = gGrid->GetHomeDirectory();
401 workdir += fGridWorkingDir;
402 if (fInputFiles) {
403 TObjString *objstr;
404 TIter next(fInputFiles);
405 while ((objstr=(TObjString*)next())) {
406 file = workdir;
407 file += "/";
408 file += objstr->GetString();
409 // Store full lfn path
410 if (FileExists(file)) objstr->SetString(file);
411 else {
412 file = objstr->GetName();
413 if (!FileExists(objstr->GetName())) {
414 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
415 objstr->GetName(), workdir.Data());
416 return kFALSE;
417 }
418 }
419 Bool_t iscoll, isxml, usetags;
420 CheckDataType(file, iscoll, isxml, usetags);
421 if (!checked) {
422 checked = kTRUE;
423 is_collection = iscoll;
424 is_xml = isxml;
425 use_tags = usetags;
426 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
427 } else {
428 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
429 Error("CheckInputData", "Some conflict was found in the types of inputs");
430 return kFALSE;
431 }
432 }
433 }
434 }
435 // Process requested run numbers
436 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
437 // Check validity of alien data directory
438 if (!fGridDataDir.Length()) {
439 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
440 return kFALSE;
441 }
442 if (!DirectoryExists(fGridDataDir)) {
443 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
444 return kFALSE;
445 }
446 if (is_collection) {
447 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
448 return kFALSE;
449 }
450
451 if (checked && !is_xml) {
452 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
453 return kFALSE;
454 }
455 // Check validity of run number(s)
456 TObjArray *arr;
457 TObjString *os;
458 Int_t nruns = 0;
459 TString schunk, schunk2;
460 TString path;
461 if (!checked) {
462 checked = kTRUE;
463 use_tags = fDataPattern.Contains("tag");
464 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
465 }
466 if (use_tags != fDataPattern.Contains("tag")) {
467 Error("CheckInputData", "Cannot mix input files using/not using tags");
468 return kFALSE;
469 }
470 if (fRunNumbers.Length()) {
471 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
472 arr = fRunNumbers.Tokenize(" ");
473 TIter next(arr);
474 while ((os=(TObjString*)next())) {
475 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
476 if (!DirectoryExists(path)) {
477 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
478 continue;
479 }
480 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
481 TString msg = "\n##### file: ";
482 msg += path;
483 msg += " type: xml_collection;";
484 if (use_tags) msg += " using_tags: Yes";
485 else msg += " using_tags: No";
486 Info("CheckDataType", msg.Data());
487 if (fNrunsPerMaster<2) {
488 AddDataFile(Form("%s.xml", os->GetString().Data()));
489 } else {
490 nruns++;
491 if (((nruns-1)%fNrunsPerMaster) == 0) {
492 schunk = os->GetString();
493 }
494 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
495 schunk += Form("_%s.xml", os->GetString().Data());
496 AddDataFile(schunk);
497 }
498 }
499 delete arr;
500 } else {
501 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
502 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
503 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
504 if (!DirectoryExists(path)) {
505// Warning("CheckInputData", "Run number %d not found in path: <%s>", irun, path.Data());
506 continue;
507 }
508 path = Form("%s/%s%d.xml", workdir.Data(),fRunPrefix.Data(),irun);
509 TString msg = "\n##### file: ";
510 msg += path;
511 msg += " type: xml_collection;";
512 if (use_tags) msg += " using_tags: Yes";
513 else msg += " using_tags: No";
514 Info("CheckDataType", msg.Data());
515 if (fNrunsPerMaster<2) {
516 AddDataFile(Form("%s%d.xml",fRunPrefix.Data(),irun));
517 } else {
518 nruns++;
519 if (((nruns-1)%fNrunsPerMaster) == 0) {
520 schunk = Form("%s%d", fRunPrefix.Data(),irun);
521 }
522 schunk2 = Form("_%s%d.xml", fRunPrefix.Data(), irun);
523 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
524 schunk += schunk2;
525 AddDataFile(schunk);
526 }
527 }
528 if (!fInputFiles) {
529 schunk += schunk2;
530 AddDataFile(schunk);
531 }
532 }
533 return kTRUE;
534}
535
536//______________________________________________________________________________
537Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
538{
539// Create dataset for the grid data directory + run number.
540 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
541 if (!Connect()) {
542 Error("CreateDataset", "Cannot create dataset with no grid connection");
543 return kFALSE;
544 }
545
546 // Cd workspace
547 CdWork();
548 TString workdir = gGrid->GetHomeDirectory();
549 workdir += fGridWorkingDir;
550
551 // Compose the 'find' command arguments
552 TString command;
553 TString options = "-x collection ";
554 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
555 TString conditions = "";
556
557 TString file;
558 TString path;
559 Int_t nruns = 0;
560 TString schunk, schunk2;
561 TGridCollection *cbase=0, *cadd=0;
562 if (!fRunNumbers.Length() && !fRunRange[0]) {
563 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
564 // Make a single data collection from data directory.
565 path = fGridDataDir;
566 if (!DirectoryExists(path)) {
567 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
568 return kFALSE;
569 }
570// CdWork();
571 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
572 else file = Form("%s.xml", gSystem->BaseName(path));
573 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest)) {
574 command = "find ";
575 command += options;
576 command += path;
577 command += " ";
578 command += pattern;
579 command += conditions;
580 printf("command: %s\n", command.Data());
581 TGridResult *res = gGrid->Command(command);
582 if (res) delete res;
583 // Write standard output to file
584 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
585 }
586 if (!TestBit(AliAnalysisGrid::kTest) && !FileExists(file)) {
587 // Copy xml file to alien space
588 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
589 if (!FileExists(file)) {
590 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
591 return kFALSE;
592 }
593 // Update list of files to be processed.
594 }
595 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
596 return kTRUE;
597 }
598 // Several runs
599 if (fRunNumbers.Length()) {
600 TObjArray *arr = fRunNumbers.Tokenize(" ");
601 TObjString *os;
602 TIter next(arr);
603 while ((os=(TObjString*)next())) {
604 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
605 if (!DirectoryExists(path)) continue;
606// CdWork();
607 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
608 else file = Form("%s.xml", os->GetString().Data());
609 // If local collection file does not exist, create it via 'find' command.
610 if (gSystem->AccessPathName(file)) {
611 command = "find ";
612 command += options;
613 command += path;
614 command += pattern;
615 command += conditions;
616 TGridResult *res = gGrid->Command(command);
617 if (res) delete res;
618 // Write standard output to file
619 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
620 }
621 if (TestBit(AliAnalysisGrid::kTest)) break;
622 // Check if there is one run per master job.
623 if (fNrunsPerMaster<2) {
624 if (FileExists(file)) {
625 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
626 continue;
627 }
628 // Copy xml file to alien space
629 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
630 if (!FileExists(file)) {
631 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
632 delete arr;
633 return kFALSE;
634 }
635 } else {
636 nruns++;
637 if (((nruns-1)%fNrunsPerMaster) == 0) {
638 schunk = os->GetString();
639 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
640 } else {
641 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
642 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
643 cbase->Add(cadd);
644 delete cadd;
645 }
646 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
647 continue;
648 }
649 schunk += Form("_%s.xml", os->GetString().Data());
650 if (FileExists(schunk)) {
651 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
652 continue;
653 }
654 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
655 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
656 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
657 if (!FileExists(schunk)) {
658 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
659 delete arr;
660 return kFALSE;
661 }
662 }
663 }
664 delete arr;
665 } else {
666 // Process a full run range.
667 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
668 path = Form("%s/%s%d ", fGridDataDir.Data(), fRunPrefix.Data(), irun);
669 if (!DirectoryExists(path)) continue;
670// CdWork();
671 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
672 else file = Form("%s%d.xml", fRunPrefix.Data(), irun);
673 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
674 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
675// gGrid->Rm(file);
676 continue;
677 }
678 // If local collection file does not exist, create it via 'find' command.
679 if (gSystem->AccessPathName(file)) {
680 command = "find ";
681 command += options;
682 command += path;
683 command += pattern;
684 command += conditions;
685 TGridResult *res = gGrid->Command(command);
686 if (res) delete res;
687 // Write standard output to file
688 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
689 }
690 if (TestBit(AliAnalysisGrid::kTest)) break;
691 // Check if there is one run per master job.
692 if (fNrunsPerMaster<2) {
693 if (FileExists(file)) {
694 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
695 continue;
696 }
697 // Copy xml file to alien space
698 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
699 if (!FileExists(file)) {
700 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
701 return kFALSE;
702 }
703 } else {
704 nruns++;
705 // Check if the collection for the chunk exist locally.
706 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
707 if (FileExists(fInputFiles->At(nchunk)->GetName())) continue;
708 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
709 if (((nruns-1)%fNrunsPerMaster) == 0) {
710 schunk = Form("%s%d", fRunPrefix.Data(), irun);
711 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
712 } else {
713 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
714 cbase->Add(cadd);
715 delete cadd;
716 }
717 schunk2 = Form("%s_%s%d.xml", schunk.Data(), fRunPrefix.Data(), irun);
718 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
719 continue;
720 }
721 schunk = schunk2;
722 if (FileExists(schunk)) {
723 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
724 continue;
725 }
726 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
727 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
728 if (FileExists(schunk)) {
729 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
730 continue;
731 }
732 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
733 if (!FileExists(schunk)) {
734 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
735 return kFALSE;
736 }
737 }
738 }
739 }
740 return kTRUE;
741}
742
743//______________________________________________________________________________
744Bool_t AliAnalysisAlien::CreateJDL()
745{
746// Generate a JDL file according to current settings. The name of the file is
747// specified by fJDLName.
748 Bool_t error = kFALSE;
749 TObjArray *arr = 0;
750 Bool_t copy = kTRUE;
751 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
752 Bool_t generate = kTRUE;
753 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
754 if (!Connect()) {
755 Error("CreateJDL", "Alien connection required");
756 return kFALSE;
757 }
758 // Check validity of alien workspace
759 CdWork();
760 TString workdir = gGrid->GetHomeDirectory();
761 workdir += fGridWorkingDir;
762 if (generate) {
763 TObjString *os;
764 if (!fInputFiles) {
765 Error("CreateJDL()", "Define some input files for your analysis.");
766 error = kTRUE;
767 }
768 // Compose list of input files
769 // Check if output files were defined
770 if (!fOutputFiles.Length()) {
771 Error("CreateJDL", "You must define at least one output file");
772 error = kTRUE;
773 }
774 // Check if an output directory was defined and valid
775 if (!fGridOutputDir.Length()) {
776 Error("CreateJDL", "You must define AliEn output directory");
777 error = kTRUE;
778 } else {
779 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
780 if (!DirectoryExists(fGridOutputDir)) {
781 if (gGrid->Mkdir(fGridOutputDir)) {
782 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
783 } else {
784 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
785 // error = kTRUE;
786 }
787 }
788 gGrid->Cd(workdir);
789 }
790 // Exit if any error up to now
791 if (error) return kFALSE;
792 // Set JDL fields
793 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
794 fGridJDL->SetExecutable(fExecutable);
795 if (!fArguments.IsNull())
796 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
797// fGridJDL->SetTTL((UInt_t)fTTL);
798 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
799 if (fMaxInitFailed > 0)
800 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
801 if (fSplitMaxInputFileNumber > 0)
802 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
803 if (fSplitMode.Length())
804 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
805// fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
806 if (fAliROOTVersion.Length())
807 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
808 if (fROOTVersion.Length())
809 fGridJDL->AddToPackages("ROOT", fROOTVersion);
810 if (fAPIVersion.Length())
811 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
812 if (!fExternalPackages.IsNull()) {
813 arr = fExternalPackages.Tokenize(" ");
814 TIter next(arr);
815 while ((os=(TObjString*)next())) {
816 TString pkgname = os->GetString();
817 Int_t index = pkgname.Index("::");
818 TString pkgversion = pkgname(index+2, pkgname.Length());
819 pkgname.Remove(index);
820 fGridJDL->AddToPackages(pkgname, pkgversion);
821 }
822 delete arr;
823 }
824 fGridJDL->SetInputDataListFormat(fInputFormat);
825 fGridJDL->SetInputDataList("wn.xml");
826 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
827 TString analysisFile = fExecutable;
828 analysisFile.ReplaceAll(".sh", ".root");
829 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
830 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
831 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
832 if (fAdditionalLibs.Length()) {
833 arr = fAdditionalLibs.Tokenize(" ");
834 TIter next(arr);
835 while ((os=(TObjString*)next())) {
836 if (os->GetString().Contains(".so")) continue;
837 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
838 }
839 delete arr;
840 }
841 if (fPackages) {
842 TIter next(fPackages);
843 TObject *obj;
844 while ((obj=next()))
845 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
846 }
847 if (fOutputArchive.Length()) {
848 arr = fOutputArchive.Tokenize(" ");
849 TIter next(arr);
850 while ((os=(TObjString*)next()))
851 if (!os->GetString().Contains("@") && fCloseSE.Length())
852 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
853 else
854 fGridJDL->AddToOutputArchive(os->GetString());
855 delete arr;
856 }
857 arr = fOutputFiles.Tokenize(" ");
858 TIter next(arr);
859 while ((os=(TObjString*)next())) {
860 // Ignore ouputs in jdl that are also in outputarchive
861 TString sout = os->GetString();
862 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
863 if (fOutputArchive.Contains(sout)) continue;
864 if (!os->GetString().Contains("@") && fCloseSE.Length())
865 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
866 else
867 fGridJDL->AddToOutputSandbox(os->GetString());
868 }
869 delete arr;
870// fGridJDL->SetPrice((UInt_t)fPrice);
871 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
872 TString validationScript = fExecutable;
873 validationScript.ReplaceAll(".sh", "_validation.sh");
874 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()));
875 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
876 // Write a jdl with 2 input parameters: collection name and output dir name.
877 WriteJDL(copy);
878 }
879 // Copy jdl to grid workspace
880 if (copy) {
881 // Check if an output directory was defined and valid
882 if (!fGridOutputDir.Length()) {
883 Error("CreateJDL", "You must define AliEn output directory");
884 return kFALSE;
885 } else {
886 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
887 if (!DirectoryExists(fGridOutputDir)) {
888 if (gGrid->Mkdir(fGridOutputDir)) {
889 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
890 } else {
891 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
892 return kFALSE;
893 }
894 }
895 gGrid->Cd(workdir);
896 }
897 if (TestBit(AliAnalysisGrid::kSubmit)) {
898 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
899 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
900 if (fProductionMode)
901 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
902 if (FileExists(locjdl)) gGrid->Rm(locjdl);
903 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
904 }
905 if (fAdditionalLibs.Length()) {
906 arr = fAdditionalLibs.Tokenize(" ");
907 TObjString *os;
908 TIter next(arr);
909 while ((os=(TObjString*)next())) {
910 if (os->GetString().Contains(".so")) continue;
911 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
912 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
913 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
914 }
915 delete arr;
916 }
917 if (fPackages) {
918 TIter next(fPackages);
919 TObject *obj;
920 while ((obj=next())) {
921 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
922 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
923 }
924 }
925 }
926 return kTRUE;
927}
928
929//______________________________________________________________________________
930Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
931{
932// Writes one or more JDL's corresponding to findex. If findex is negative,
933// all run numbers are considered in one go (jdl). For non-negative indices
934// they correspond to the indices in the array fInputFiles.
935 if (!fInputFiles) return kFALSE;
936 TObjString *os;
937 TString workdir = gGrid->GetHomeDirectory();
938 workdir += fGridWorkingDir;
939
940 if (!fRunNumbers.Length() && !fRunRange[0]) {
941 // One jdl with no parameters in case input data is specified by name.
942 TIter next(fInputFiles);
943 while ((os=(TObjString*)next()))
944 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
945 if (!fOutputSingle.IsNull())
946 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
947 else
948 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()));
949 } else {
950 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
951 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()));
952 if (!fOutputSingle.IsNull()) {
953 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()));
954 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()));
955 } else {
956 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()));
957 }
958 }
959
960
961 // Generate the JDL as a string
962 TString sjdl = fGridJDL->Generate();
963 Int_t index;
964 index = sjdl.Index("Executable");
965 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
966 index = sjdl.Index("Split ");
967 if (index >= 0) sjdl.Insert(index, "\n# We split per SE or file\n");
968 index = sjdl.Index("SplitMaxInputFileNumber");
969 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
970 index = sjdl.Index("InputDataCollection");
971 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
972 index = sjdl.Index("InputFile");
973 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
974 index = sjdl.Index("InputDataList ");
975 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
976 index = sjdl.Index("InputDataListFormat");
977 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
978 index = sjdl.Index("Price");
979 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
980 index = sjdl.Index("Requirements");
981 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
982 index = sjdl.Index("Packages");
983 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
984 index = sjdl.Index("User =");
985 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
986 index = sjdl.Index("TTL");
987 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
988 index = sjdl.Index("OutputFile");
989 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
990 index = sjdl.Index("OutputDir");
991 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
992 index = sjdl.Index("OutputArchive");
993 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
994 index = sjdl.Index("MaxInitFailed");
995 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
996 index = sjdl.Index("MasterResubmitThreshold");
997 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
998 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
999 index = sjdl.Index("Validationcommand");
1000 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
1001 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1002 sjdl.ReplaceAll("(member", "\n (member");
1003 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1004 sjdl.ReplaceAll("{", "{\n ");
1005 sjdl.ReplaceAll("};", "\n};");
1006 sjdl.ReplaceAll("{\n \n", "{\n");
1007 sjdl.ReplaceAll("\n\n", "\n");
1008 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
1009 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1010 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
1011 index = sjdl.Index("JDLVariables");
1012 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
1013 // Write jdl to file
1014 ofstream out;
1015 out.open(fJDLName.Data(), ios::out);
1016 if (out.bad()) {
1017 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
1018 return kFALSE;
1019 }
1020 out << sjdl << endl;
1021
1022 // Copy jdl to grid workspace
1023 if (!copy) {
1024 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1025 } else {
1026 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1027 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
1028 if (fProductionMode)
1029 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
1030 if (FileExists(locjdl)) gGrid->Rm(locjdl);
1031 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1032 }
1033 return kTRUE;
1034}
1035
1036//______________________________________________________________________________
1037Bool_t AliAnalysisAlien::FileExists(const char *lfn)
1038{
1039// Returns true if file exists.
1040 if (!gGrid) return kFALSE;
1041 TGridResult *res = gGrid->Ls(lfn);
1042 if (!res) return kFALSE;
1043 TMap *map = dynamic_cast<TMap*>(res->At(0));
1044 if (!map) {
1045 delete res;
1046 return kFALSE;
1047 }
1048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1049 if (!objs || !objs->GetString().Length()) {
1050 delete res;
1051 return kFALSE;
1052 }
1053 delete res;
1054 return kTRUE;
1055}
1056
1057//______________________________________________________________________________
1058Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1059{
1060// Returns true if directory exists. Can be also a path.
1061 if (!gGrid) return kFALSE;
1062 // Check if dirname is a path
1063 TString dirstripped = dirname;
1064 dirstripped = dirstripped.Strip();
1065 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1066 TString dir = gSystem->BaseName(dirstripped);
1067 dir += "/";
1068 TString path = gSystem->DirName(dirstripped);
1069 TGridResult *res = gGrid->Ls(path, "-F");
1070 if (!res) return kFALSE;
1071 TIter next(res);
1072 TMap *map;
1073 TObject *obj;
1074 while ((map=dynamic_cast<TMap*>(next()))) {
1075 obj = map->GetValue("name");
1076 if (!obj) break;
1077 if (dir == obj->GetName()) {
1078 delete res;
1079 return kTRUE;
1080 }
1081 }
1082 delete res;
1083 return kFALSE;
1084}
1085
1086//______________________________________________________________________________
1087void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
1088{
1089// Check input data type.
1090 is_collection = kFALSE;
1091 is_xml = kFALSE;
1092 use_tags = kFALSE;
1093 if (!gGrid) {
1094 Error("CheckDataType", "No connection to grid");
1095 return;
1096 }
1097 is_collection = IsCollection(lfn);
1098 TString msg = "\n##### file: ";
1099 msg += lfn;
1100 if (is_collection) {
1101 msg += " type: raw_collection;";
1102 // special treatment for collections
1103 is_xml = kFALSE;
1104 // check for tag files in the collection
1105 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1106 if (!res) {
1107 msg += " using_tags: No (unknown)";
1108 Info("CheckDataType", msg.Data());
1109 return;
1110 }
1111 const char* typeStr = res->GetKey(0, "origLFN");
1112 if (!typeStr || !strlen(typeStr)) {
1113 msg += " using_tags: No (unknown)";
1114 Info("CheckDataType", msg.Data());
1115 return;
1116 }
1117 TString file = typeStr;
1118 use_tags = file.Contains(".tag");
1119 if (use_tags) msg += " using_tags: Yes";
1120 else msg += " using_tags: No";
1121 Info("CheckDataType", msg.Data());
1122 return;
1123 }
1124 TString slfn(lfn);
1125 slfn.ToLower();
1126 is_xml = slfn.Contains(".xml");
1127 if (is_xml) {
1128 // Open xml collection and check if there are tag files inside
1129 msg += " type: xml_collection;";
1130 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1131 if (!coll) {
1132 msg += " using_tags: No (unknown)";
1133 Info("CheckDataType", msg.Data());
1134 return;
1135 }
1136 TMap *map = coll->Next();
1137 if (!map) {
1138 msg += " using_tags: No (unknown)";
1139 Info("CheckDataType", msg.Data());
1140 return;
1141 }
1142 map = (TMap*)map->GetValue("");
1143 TString file;
1144 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1145 use_tags = file.Contains(".tag");
1146 delete coll;
1147 if (use_tags) msg += " using_tags: Yes";
1148 else msg += " using_tags: No";
1149 Info("CheckDataType", msg.Data());
1150 return;
1151 }
1152 use_tags = slfn.Contains(".tag");
1153 if (slfn.Contains(".root")) msg += " type: root file;";
1154 else msg += " type: unhnown file;";
1155 if (use_tags) msg += " using_tags: Yes";
1156 else msg += " using_tags: No";
1157 Info("CheckDataType", msg.Data());
1158}
1159
1160//______________________________________________________________________________
1161void AliAnalysisAlien::EnablePackage(const char *package)
1162{
1163// Enables a par file supposed to exist in the current directory.
1164 TString pkg(package);
1165 pkg.ReplaceAll(".par", "");
1166 pkg += ".par";
1167 if (gSystem->AccessPathName(pkg)) {
1168 Error("EnablePackage", "Package %s not found", pkg.Data());
1169 return;
1170 }
1171 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1172 Info("EnablePackage", "AliEn plugin will use .par packages");
1173 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1174 if (!fPackages) {
1175 fPackages = new TObjArray();
1176 fPackages->SetOwner();
1177 }
1178 fPackages->Add(new TObjString(pkg));
1179}
1180
1181//______________________________________________________________________________
1182const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1183{
1184// Get job status for all jobs with jobid>jobidstart.
1185 static char mstatus[20];
1186 mstatus[0] = '\0';
1187 nrunning = 0;
1188 nwaiting = 0;
1189 nerror = 0;
1190 ndone = 0;
1191 TGridJobStatusList *list = gGrid->Ps("");
1192 if (!list) return mstatus;
1193 Int_t nentries = list->GetSize();
1194 TGridJobStatus *status;
1195 Int_t pid;
1196 for (Int_t ijob=0; ijob<nentries; ijob++) {
1197 status = (TGridJobStatus *)list->At(ijob);
1198 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)0x%lx)->GetKey(\"queueId\"));", (ULong_t)status));
1199 if (pid<jobidstart) continue;
1200 if (pid == lastid) {
1201 gROOT->ProcessLine(Form("sprintf((char*)0x%lx,((TAlienJobStatus*)0x%lx)->GetKey(\"status\"));",(ULong_t)mstatus, (ULong_t)status));
1202 }
1203 switch (status->GetStatus()) {
1204 case TGridJobStatus::kWAITING:
1205 nwaiting++; break;
1206 case TGridJobStatus::kRUNNING:
1207 nrunning++; break;
1208 case TGridJobStatus::kABORTED:
1209 case TGridJobStatus::kFAIL:
1210 case TGridJobStatus::kUNKNOWN:
1211 nerror++; break;
1212 case TGridJobStatus::kDONE:
1213 ndone++;
1214 }
1215 }
1216 list->Delete();
1217 delete list;
1218 return mstatus;
1219}
1220
1221//______________________________________________________________________________
1222Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1223{
1224// Returns true if file is a collection. Functionality duplicated from
1225// TAlien::Type() because we don't want to directly depend on TAlien.
1226 if (!gGrid) {
1227 Error("IsCollection", "No connection to grid");
1228 return kFALSE;
1229 }
1230 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1231 if (!res) return kFALSE;
1232 const char* typeStr = res->GetKey(0, "type");
1233 if (!typeStr || !strlen(typeStr)) return kFALSE;
1234 if (!strcmp(typeStr, "collection")) return kTRUE;
1235 delete res;
1236 return kFALSE;
1237}
1238
1239//______________________________________________________________________________
1240Bool_t AliAnalysisAlien::IsSingleOutput() const
1241{
1242// Check if single-ouput option is on.
1243 return (!fOutputSingle.IsNull());
1244}
1245
1246//______________________________________________________________________________
1247void AliAnalysisAlien::Print(Option_t *) const
1248{
1249// Print current plugin settings.
1250 printf("### AliEn analysis plugin current settings ###\n");
1251 printf("= Production mode:______________________________ %d\n", fProductionMode);
1252 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1253 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1254 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1255 if (fUser.Length())
1256 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1257 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1258 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1259 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1260 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1261 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1262 if (fRunNumbers.Length())
1263 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1264 if (fRunRange[0])
1265 printf("= Run range to be processed: ___________________ %s%d-%s%d\n", fRunPrefix.Data(), fRunRange[0], fRunPrefix.Data(), fRunRange[1]);
1266 if (!fRunRange[0] && !fRunNumbers.Length()) {
1267 TIter next(fInputFiles);
1268 TObject *obj;
1269 TString list;
1270 while ((obj=next())) list += obj->GetName();
1271 printf("= Input files to be processed: _________________ %s\n", list.Data());
1272 }
1273 if (TestBit(AliAnalysisGrid::kTest))
1274 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1275 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1276 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1277 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1278 printf("=====================================================================\n");
1279 printf("= Job price: ___________________________________ %d\n", fPrice);
1280 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1281 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1282 if (fMaxInitFailed>0)
1283 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1284 if (fMasterResubmitThreshold>0)
1285 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
1286 if (fNrunsPerMaster>0)
1287 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1288 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1289 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1290 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
1291 if (fArguments.Length())
1292 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1293 if (fExecutableArgs.Length())
1294 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
1295 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1296 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1297 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1298 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1299 if (fDatasetName)
1300 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1301 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1302 if (fIncludePath.Data())
1303 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1304 if (fCloseSE.Length())
1305 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1306 if (fFriendChainName.Length())
1307 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1308 if (fPackages) {
1309 TIter next(fPackages);
1310 TObject *obj;
1311 TString list;
1312 while ((obj=next())) list += obj->GetName();
1313 printf("= Par files to be used: ________________________ %s\n", list.Data());
1314 }
1315}
1316
1317//______________________________________________________________________________
1318void AliAnalysisAlien::SetDefaults()
1319{
1320// Set default values for everything. What cannot be filled will be left empty.
1321 if (fGridJDL) delete fGridJDL;
1322 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1323 fPrice = 1;
1324 fTTL = 30000;
1325 fSplitMaxInputFileNumber = 100;
1326 fMaxInitFailed = 0;
1327 fMasterResubmitThreshold = 0;
1328 fNtestFiles = 10;
1329 fRunRange[0] = 0;
1330 fRunRange[1] = 0;
1331 fNrunsPerMaster = 1;
1332 fMaxMergeFiles = 100;
1333 fRunNumbers = "";
1334 fExecutable = "analysis.sh";
1335 fExecutableCommand = "root -b -q";
1336 fArguments = "";
1337 fExecutableArgs = "";
1338 fAnalysisMacro = "myAnalysis.C";
1339 fAnalysisSource = "";
1340 fAdditionalLibs = "";
1341 fSplitMode = "se";
1342 fAPIVersion = "";
1343 fROOTVersion = "";
1344 fAliROOTVersion = "";
1345 fUser = ""; // Your alien user name
1346 fGridWorkingDir = "";
1347 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1348 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
1349 fFriendChainName = "";
1350 fGridOutputDir = "output";
1351 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
1352 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1353 fInputFormat = "xml-single";
1354 fJDLName = "analysis.jdl";
1355 fJobTag = "Automatically generated analysis JDL";
1356 fMergeExcludes = "";
1357}
1358
1359//______________________________________________________________________________
1360Bool_t AliAnalysisAlien::MergeOutputs()
1361{
1362// Merge analysis outputs existing in the AliEn space.
1363 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
1364 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1365 if (!Connect()) {
1366 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
1367 return kFALSE;
1368 }
1369 // Get the output path
1370 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
1371 if (!DirectoryExists(fGridOutputDir)) {
1372 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
1373 return kFALSE;
1374 }
1375 if (!fOutputFiles.Length()) {
1376 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
1377 return kFALSE;
1378 }
1379 TObjArray *list = fOutputFiles.Tokenize(" ");
1380 TIter next(list);
1381 TObjString *str;
1382 TString command;
1383 TString output_file;
1384 TString output_chunk;
1385 TString previous_chunk;
1386 Int_t count_chunk = 0;
1387 Int_t count_zero = fMaxMergeFiles;
1388 Bool_t merged = kTRUE;
1389 while((str=(TObjString*)next())) {
1390 output_file = str->GetString();
1391 Int_t index = output_file.Index("@");
1392 if (index > 0) output_file.Remove(index);
1393 // Skip already merged outputs
1394 if (!gSystem->AccessPathName(output_file)) {
1395 Info("MergeOutputs", "Output file <%s> found. Not merging again.", output_file.Data());
1396 continue;
1397 }
1398 if (fMergeExcludes.Length() &&
1399 fMergeExcludes.Contains(output_file.Data())) continue;
1400 // Perform a 'find' command in the output directory, looking for registered outputs
1401 command = Form("find %s/ *%s", fGridOutputDir.Data(), output_file.Data());
1402 printf("command: %s\n", command.Data());
1403 TGridResult *res = gGrid->Command(command);
1404 if (!res) continue;
1405 TFileMerger *fm = 0;
1406 TIter nextmap(res);
1407 TMap *map = 0;
1408 previous_chunk = "";
1409 count_chunk = 0;
1410 // Check if there is a merge operation to resume
1411 output_chunk = output_file;
1412 output_chunk.ReplaceAll(".root", "_*.root");
1413 if (!gSystem->Exec(Form("ls %s", output_chunk.Data()))) {
1414 while (1) {
1415 for (Int_t counter=0; counter<fMaxMergeFiles; counter++) map = (TMap*)nextmap();
1416 if (!map) {
1417 Error("MergeOutputs", "Cannot resume merging for <%s>, nentries=%d", output_file.Data(), res->GetSize());
1418 delete res;
1419 return kFALSE;
1420 }
1421 output_chunk = output_file;
1422 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1423 printf("%s\n", output_chunk.Data());
1424 count_chunk++;
1425 if (gSystem->AccessPathName(output_chunk)) continue;
1426 // Merged file with chunks up to <count_chunk> found
1427 printf("Resume merging of <%s> from <%s>\n", output_file.Data(), output_chunk.Data());
1428 previous_chunk = output_chunk;
1429 break;
1430 }
1431 }
1432 count_zero = fMaxMergeFiles;
1433 while ((map=(TMap*)nextmap())) {
1434 // Loop 'find' results and get next LFN
1435 if (count_zero == fMaxMergeFiles) {
1436 // First file in chunk - create file merger and add previous chunk if any.
1437 fm = new TFileMerger(kFALSE);
1438 fm->SetFastMethod(kTRUE);
1439 if (previous_chunk.Length()) fm->AddFile(previous_chunk.Data());
1440 output_chunk = output_file;
1441 output_chunk.ReplaceAll(".root", Form("_%04d.root", count_chunk));
1442 }
1443 // If last file found, put merged results in the output file
1444 if (map == res->Last()) output_chunk = output_file;
1445 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
1446 if (!objs || !objs->GetString().Length()) {
1447 // Nothing found - skip this output
1448 delete res;
1449 delete fm;
1450 break;
1451 }
1452 // Add file to be merged and decrement chunk counter.
1453 fm->AddFile(objs->GetString());
1454 count_zero--;
1455 if (count_zero==0 || map == res->Last()) {
1456 fm->OutputFile(output_chunk);
1457 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
1458 // Nothing found - skip this output
1459 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
1460 delete res;
1461 delete fm;
1462 break;
1463 }
1464 // Merge the outputs, then go to next chunk
1465 if (!fm->Merge()) {
1466 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
1467 delete res;
1468 delete fm;
1469 merged = kFALSE;
1470 break;
1471 } else {
1472 Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), output_chunk.Data());
1473 gSystem->Unlink(previous_chunk);
1474 }
1475 if (map == res->Last()) {
1476 delete res;
1477 delete fm;
1478 break;
1479 }
1480 count_chunk++;
1481 count_zero = fMaxMergeFiles;
1482 previous_chunk = output_chunk;
1483 }
1484 }
1485 }
1486 if (!merged) {
1487 Error("MergeOutputs", "Terminate() will NOT be executed");
1488 }
1489 return merged;
1490}
1491
1492//______________________________________________________________________________
1493void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
1494{
1495// Use the output files connected to output containers from the analysis manager
1496// rather than the files defined by SetOutputFiles
1497 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
1498 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
1499 analysis manager");
1500 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
1501}
1502
1503//______________________________________________________________________________
1504Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
1505{
1506// Start remote grid analysis.
1507
1508 // Check if output files have to be taken from the analysis manager
1509 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1510 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1511 if (!mgr || !mgr->IsInitialized()) {
1512 Error("StartAnalysis", "You need an initialized analysis manager for this");
1513 return kFALSE;
1514 }
1515 fOutputFiles = "";
1516 TIter next(mgr->GetOutputs());
1517 AliAnalysisDataContainer *output;
1518 while ((output=(AliAnalysisDataContainer*)next())) {
1519 const char *filename = output->GetFileName();
1520 if (!(strcmp(filename, "default"))) {
1521 if (!mgr->GetOutputEventHandler()) continue;
1522 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
1523 }
1524 if (fOutputFiles.Contains(filename)) continue;
1525 if (fOutputFiles.Length()) fOutputFiles += " ";
1526 fOutputFiles += filename;
1527 }
1528 // Add extra files registered to the analysis manager
1529 if (mgr->GetExtraFiles().Length()) {
1530 if (fOutputFiles.Length()) fOutputFiles += " ";
1531 fOutputFiles += mgr->GetExtraFiles();
1532 }
1533 }
1534// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
1535 if (TestBit(AliAnalysisGrid::kOffline)) {
1536 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
1537 \n there nor any job run. You can revise the JDL and analysis \
1538 \n macro then run the same in \"submit\" mode.");
1539 } else if (TestBit(AliAnalysisGrid::kTest)) {
1540 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
1541 \n dataset.");
1542 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
1543 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
1544 \n space and job submitted.");
1545 } else if (TestBit(AliAnalysisGrid::kMerge)) {
1546 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
1547 return kTRUE;
1548 } else {
1549 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
1550 }
1551
1552 if (!Connect()) {
1553 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
1554 return kFALSE;
1555 }
1556 Print();
1557 if (!CheckInputData()) {
1558 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
1559 return kFALSE;
1560 }
1561 CreateDataset(fDataPattern);
1562 WriteAnalysisFile();
1563 WriteAnalysisMacro();
1564 WriteExecutable();
1565 WriteValidationScript();
1566 if (!CreateJDL()) return kFALSE;
1567 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
1568 if (TestBit(AliAnalysisGrid::kTest)) {
1569 // Locally testing the analysis
1570 Info("StartAnalysis", "\n_______________________________________________________________________ \
1571 \n Running analysis script in a daughter shell as on a worker node \
1572 \n_______________________________________________________________________");
1573 TObjArray *list = fOutputFiles.Tokenize(" ");
1574 TIter next(list);
1575 TObjString *str;
1576 TString output_file;
1577 while((str=(TObjString*)next())) {
1578 output_file = str->GetString();
1579 Int_t index = output_file.Index("@");
1580 if (index > 0) output_file.Remove(index);
1581 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
1582 }
1583 delete list;
1584 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1585 TString validationScript = fExecutable;
1586 validationScript.ReplaceAll(".sh", "_validation.sh");
1587 gSystem->Exec(Form("bash %s",validationScript.Data()));
1588// gSystem->Exec("cat stdout");
1589 return kFALSE;
1590 }
1591 // Check if submitting is managed by LPM manager
1592 if (fProductionMode) {
1593 TString prodfile = fJDLName;
1594 prodfile.ReplaceAll(".jdl", ".prod");
1595 WriteProductionFile(prodfile);
1596 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
1597 return kFALSE;
1598 }
1599 // Submit AliEn job(s)
1600 gGrid->Cd(fGridOutputDir);
1601 TGridResult *res;
1602 TString jobID = "";
1603 if (!fRunNumbers.Length() && !fRunRange[0]) {
1604 // Submit a given xml or a set of runs
1605 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1606 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1607 if (res) {
1608 const char *cjobId = res->GetKey(0,"jobId");
1609 if (!cjobId) {
1610 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1611 return kFALSE;
1612 } else {
1613 Info("StartAnalysis", "\n_______________________________________________________________________ \
1614 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1615 \n_______________________________________________________________________",
1616 fJDLName.Data(), cjobId);
1617 jobID = cjobId;
1618 }
1619 delete res;
1620 }
1621 } else {
1622 // Submit for a range of enumeration of runs.
1623 Submit();
1624 }
1625
1626 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1627 \n You may exit at any time and terminate the job later using the option <terminate> \
1628 \n ##################################################################################", jobID.Data());
1629 gSystem->Exec("aliensh");
1630 return kTRUE;
1631}
1632
1633//______________________________________________________________________________
1634void AliAnalysisAlien::Submit()
1635{
1636// Submit all master jobs.
1637 Int_t nmasterjobs = fInputFiles->GetEntries();
1638 Long_t tshoot = gSystem->Now();
1639 if (!fNsubmitted) SubmitNext();
1640 while (fNsubmitted < nmasterjobs) {
1641 Long_t now = gSystem->Now();
1642 if ((now-tshoot)>30000) {
1643 tshoot = now;
1644 SubmitNext();
1645 }
1646 }
1647}
1648
1649//______________________________________________________________________________
1650void AliAnalysisAlien::SubmitNext()
1651{
1652// Submit next bunch of master jobs if the queue is free.
1653 static Bool_t iscalled = kFALSE;
1654 static Int_t firstmaster = 0;
1655 static Int_t lastmaster = 0;
1656 static Int_t npermaster = 0;
1657 if (iscalled) return;
1658 iscalled = kTRUE;
1659 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
1660 Int_t ntosubmit = 0;
1661 TGridResult *res;
1662 TString jobID = "";
1663 if (!fNsubmitted) ntosubmit = 1;
1664 else {
1665 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
1666 printf("=== master %d: %s\n", lastmaster, status.Data());
1667 // If last master not split, just return
1668 if (status != "SPLIT") {iscalled = kFALSE; return;}
1669 // No more than 100 waiting jobs
1670 if (nwaiting>100) {iscalled = kFALSE; return;}
1671 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
1672 if (npermaster) ntosubmit = (100-nwaiting)/npermaster;
1673 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
1674 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
1675 }
1676 Int_t nmasterjobs = fInputFiles->GetEntries();
1677 for (Int_t i=0; i<ntosubmit; i++) {
1678 // Submit for a range of enumeration of runs.
1679 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return;}
1680 TString query;
1681 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
1682 runOutDir.ReplaceAll(".xml", "");
1683 if (fOutputToRunNo)
1684 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
1685 else
1686 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
1687 printf("********* %s\n",query.Data());
1688 res = gGrid->Command(query);
1689 if (res) {
1690 TString cjobId1 = res->GetKey(0,"jobId");
1691 if (!cjobId1.Length()) {
1692 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1693 iscalled = kFALSE;
1694 return;
1695 } else {
1696 Info("StartAnalysis", "\n_______________________________________________________________________ \
1697 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
1698 \n_______________________________________________________________________",
1699 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
1700 jobID += cjobId1;
1701 jobID += " ";
1702 lastmaster = cjobId1.Atoi();
1703 if (!firstmaster) firstmaster = lastmaster;
1704 fNsubmitted++;
1705 }
1706 delete res;
1707 }
1708 }
1709 iscalled = kFALSE;
1710}
1711
1712//______________________________________________________________________________
1713void AliAnalysisAlien::WriteAnalysisFile()
1714{
1715// Write current analysis manager into the file <analysisFile>
1716 TString analysisFile = fExecutable;
1717 analysisFile.ReplaceAll(".sh", ".root");
1718 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1719 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1720 if (!mgr || !mgr->IsInitialized()) {
1721 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1722 return;
1723 }
1724 // Check analysis type
1725 TObject *handler;
1726 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1727 handler = (TObject*)mgr->GetInputEventHandler();
1728 if (handler) {
1729 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1730 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1731 }
1732 TDirectory *cdir = gDirectory;
1733 TFile *file = TFile::Open(analysisFile, "RECREATE");
1734 if (file) {
1735 // Skip task Terminate calls for the grid job
1736 mgr->SetSkipTerminate(kTRUE);
1737 // Unless merging makes no sense
1738 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
1739 mgr->Write();
1740 delete file;
1741 // Enable termination for local jobs
1742 mgr->SetSkipTerminate(kFALSE);
1743 }
1744 if (cdir) cdir->cd();
1745 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
1746 }
1747 Bool_t copy = kTRUE;
1748 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1749 if (copy) {
1750 CdWork();
1751 TString workdir = gGrid->GetHomeDirectory();
1752 workdir += fGridWorkingDir;
1753 Info("CreateJDL", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
1754 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
1755 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
1756 }
1757}
1758
1759//______________________________________________________________________________
1760void AliAnalysisAlien::WriteAnalysisMacro()
1761{
1762// Write the analysis macro that will steer the analysis in grid mode.
1763 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1764 ofstream out;
1765 out.open(fAnalysisMacro.Data(), ios::out);
1766 if (!out.good()) {
1767 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1768 return;
1769 }
1770 TString func = fAnalysisMacro;
1771 TString type = "ESD";
1772 TString comment = "// Analysis using ";
1773 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1774 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1775 type = "AOD";
1776 comment += "AOD";
1777 }
1778 if (type!="AOD" && fFriendChainName!="") {
1779 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1780 return;
1781 }
1782 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1783 else comment += " data";
1784 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1785 func.ReplaceAll(".C", "");
1786 out << "void " << func.Data() << "()" << endl;
1787 out << "{" << endl;
1788 out << comment.Data() << endl;
1789 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1790 out << " TStopwatch timer;" << endl;
1791 out << " timer.Start();" << endl << endl;
1792 out << "// load base root libraries" << endl;
1793 out << " gSystem->Load(\"libTree\");" << endl;
1794 out << " gSystem->Load(\"libGeom\");" << endl;
1795 out << " gSystem->Load(\"libVMC\");" << endl;
1796 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
1797 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
1798 out << "// Load analysis framework libraries" << endl;
1799 if (!fPackages) {
1800 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1801 out << " gSystem->Load(\"libESD\");" << endl;
1802 out << " gSystem->Load(\"libAOD\");" << endl;
1803 out << " gSystem->Load(\"libANALYSIS\");" << endl;
1804 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1805 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1806 } else {
1807 TIter next(fPackages);
1808 TObject *obj;
1809 TString pkgname;
1810 Bool_t hasSTEERBase = kFALSE;
1811 Bool_t hasESD = kFALSE;
1812 Bool_t hasAOD = kFALSE;
1813 Bool_t hasANALYSIS = kFALSE;
1814 Bool_t hasANALYSISalice = kFALSE;
1815 Bool_t hasCORRFW = kFALSE;
1816 while ((obj=next())) {
1817 pkgname = obj->GetName();
1818 if (pkgname == "STEERBase" ||
1819 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1820 if (pkgname == "ESD" ||
1821 pkgname == "ESD.par") hasESD = kTRUE;
1822 if (pkgname == "AOD" ||
1823 pkgname == "AOD.par") hasAOD = kTRUE;
1824 if (pkgname == "ANALYSIS" ||
1825 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1826 if (pkgname == "ANALYSISalice" ||
1827 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1828 if (pkgname == "CORRFW" ||
1829 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
1830 }
1831 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1832 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1833 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1834 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1835 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1836 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1837 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1838 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1839 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1840 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1841 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1842 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1843 out << "// Compile other par packages" << endl;
1844 next.Reset();
1845 while ((obj=next())) {
1846 pkgname = obj->GetName();
1847 if (pkgname == "STEERBase" ||
1848 pkgname == "STEERBase.par" ||
1849 pkgname == "ESD" ||
1850 pkgname == "ESD.par" ||
1851 pkgname == "AOD" ||
1852 pkgname == "AOD.par" ||
1853 pkgname == "ANALYSIS" ||
1854 pkgname == "ANALYSIS.par" ||
1855 pkgname == "ANALYSISalice" ||
1856 pkgname == "ANALYSISalice.par" ||
1857 pkgname == "CORRFW" ||
1858 pkgname == "CORRFW.par") continue;
1859 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1860 }
1861 }
1862 out << "// include path" << endl;
1863 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1864 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
1865 if (fAdditionalLibs.Length()) {
1866 out << "// Add aditional AliRoot libraries" << endl;
1867 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1868 TIter next(list);
1869 TObjString *str;
1870 while((str=(TObjString*)next())) {
1871 if (str->GetString().Contains(".so"))
1872 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1873 }
1874 if (list) delete list;
1875 }
1876 out << endl;
1877 out << "// analysis source to be compiled at runtime (if any)" << endl;
1878 if (fAnalysisSource.Length()) {
1879 TObjArray *list = fAnalysisSource.Tokenize(" ");
1880 TIter next(list);
1881 TObjString *str;
1882 while((str=(TObjString*)next())) {
1883 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1884 }
1885 if (list) delete list;
1886 }
1887 out << endl;
1888 out << "// connect to AliEn and make the chain" << endl;
1889 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1890 if (IsUsingTags()) {
1891 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1892 } else {
1893 if(fFriendChainName!="AliAOD.VertexingHF.root") {
1894 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1895 } else {
1896 out << " // Check if the macro to create the chain was provided" << endl;
1897 out << " if (gSystem->AccessPathName(\"MakeAODInputChain.C\")) {" << endl;
1898 out << " ::Error(\"" << func.Data() << "\", \"File MakeAODInputChain.C not provided. Aborting.\");" << endl;
1899 out << " return;" << endl;
1900 out << " }" << endl;
1901 out << " gROOT->LoadMacro(\"MakeAODInputChain.C\");" << endl;
1902 out << " TChain *chain = MakeAODInputChain(\"wn.xml\",\"none\");" << endl << endl;
1903 }
1904 }
1905 out << "// read the analysis manager from file" << endl;
1906 TString analysisFile = fExecutable;
1907 analysisFile.ReplaceAll(".sh", ".root");
1908 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
1909 out << " if (!file) return;" << endl;
1910 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1911 out << " AliAnalysisManager *mgr = 0;" << endl;
1912 out << " TKey *key;" << endl;
1913 out << " while ((key=(TKey*)nextkey())) {" << endl;
1914 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1915 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1916 out << " };" << endl;
1917 out << " if (!mgr) {" << endl;
1918 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
1919 out << " return;" << endl;
1920 out << " }" << endl << endl;
1921 out << " mgr->PrintStatus();" << endl;
1922 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1923 out << " timer.Stop();" << endl;
1924 out << " timer.Print();" << endl;
1925 out << "}" << endl << endl;
1926 if (IsUsingTags()) {
1927 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1928 out << "{" << endl;
1929 out << "// Create a chain using tags from the xml file." << endl;
1930 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1931 out << " if (!coll) {" << endl;
1932 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1933 out << " return NULL;" << endl;
1934 out << " }" << endl;
1935 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1936 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1937 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1938 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1939 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1940 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1941 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1942 out << " // Check if the cuts configuration file was provided" << endl;
1943 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1944 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1945 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1946 out << " }" << endl;
1947 if (fFriendChainName=="") {
1948 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1949 } else {
1950 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1951 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1952 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1953 }
1954 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1955 out << " chain->ls();" << endl;
1956 out << " return chain;" << endl;
1957 out << "}" << endl << endl;
1958 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1959 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1960 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1961 msg += " AliLHCTagCuts *lhcCuts,\n";
1962 msg += " AliDetectorTagCuts *detCuts,\n";
1963 msg += " AliEventTagCuts *evCuts)";
1964 Info("WriteAnalysisMacro", msg.Data());
1965 }
1966 }
1967 if (!IsUsingTags() || fFriendChainName!="") {
1968 out <<"//________________________________________________________________________________" << endl;
1969 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1970 out << "{" << endl;
1971 out << "// Create a chain using url's from xml file" << endl;
1972 out << " TString treename = type;" << endl;
1973 out << " treename.ToLower();" << endl;
1974 out << " treename += \"Tree\";" << endl;
1975 out << " printf(\"***************************************\\n\");" << endl;
1976 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1977 out << " printf(\"***************************************\\n\");" << endl;
1978 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1979 out << " if (!coll) {" << endl;
1980 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1981 out << " return NULL;" << endl;
1982 out << " }" << endl;
1983 out << " TChain *chain = new TChain(treename);" << endl;
1984 if(fFriendChainName!="") {
1985 out << " TChain *chainFriend = new TChain(treename);" << endl;
1986 }
1987 out << " coll->Reset();" << endl;
1988 out << " while (coll->Next()) {" << endl;
1989 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1990 if(fFriendChainName!="") {
1991 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1992 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1993 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1994 out << " chainFriend->Add(fileFriend.Data());" << endl;
1995 }
1996 out << " }" << endl;
1997 out << " if (!chain->GetNtrees()) {" << endl;
1998 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1999 out << " return NULL;" << endl;
2000 out << " }" << endl;
2001 if(fFriendChainName!="") {
2002 out << " chain->AddFriend(chainFriend);" << endl;
2003 }
2004 out << " return chain;" << endl;
2005 out << "}" << endl << endl;
2006 }
2007 if (fPackages) {
2008 out <<"//________________________________________________________________________________" << endl;
2009 out << "Bool_t SetupPar(const char *package) {" << endl;
2010 out << "// Compile the package and set it up." << endl;
2011 out << " TString pkgdir = package;" << endl;
2012 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
2013 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
2014 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
2015 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
2016 out << " // Check for BUILD.sh and execute" << endl;
2017 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
2018 out << " printf(\"*******************************\\n\");" << endl;
2019 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
2020 out << " printf(\"*******************************\\n\");" << endl;
2021 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
2022 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
2023 out << " gSystem->ChangeDirectory(cdir);" << endl;
2024 out << " return kFALSE;" << endl;
2025 out << " }" << endl;
2026 out << " } else {" << endl;
2027 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
2028 out << " gSystem->ChangeDirectory(cdir);" << endl;
2029 out << " return kFALSE;" << endl;
2030 out << " }" << endl;
2031 out << " // Check for SETUP.C and execute" << endl;
2032 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
2033 out << " printf(\"*******************************\\n\");" << endl;
2034 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
2035 out << " printf(\"*******************************\\n\");" << endl;
2036 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
2037 out << " } else {" << endl;
2038 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
2039 out << " gSystem->ChangeDirectory(cdir);" << endl;
2040 out << " return kFALSE;" << endl;
2041 out << " }" << endl;
2042 out << " // Restore original workdir" << endl;
2043 out << " gSystem->ChangeDirectory(cdir);" << endl;
2044 out << " return kTRUE;" << endl;
2045 out << "}" << endl;
2046 }
2047 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
2048 }
2049 Bool_t copy = kTRUE;
2050 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2051 if (copy) {
2052 CdWork();
2053 TString workdir = gGrid->GetHomeDirectory();
2054 workdir += fGridWorkingDir;
2055 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
2056 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
2057 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
2058 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
2059 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
2060 }
2061 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
2062 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
2063 }
2064}
2065
2066//______________________________________________________________________________
2067void AliAnalysisAlien::WriteExecutable()
2068{
2069// Generate the alien executable script.
2070 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2071 ofstream out;
2072 out.open(fExecutable.Data(), ios::out);
2073 if (out.bad()) {
2074 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
2075 return;
2076 }
2077 out << "#!/bin/bash" << endl;
2078 out << "echo \"=========================================\"" << endl;
2079 out << "echo \"############## PATH : ##############\"" << endl;
2080 out << "echo $PATH" << endl;
2081 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
2082 out << "echo $LD_LIBRARY_PATH" << endl;
2083 out << "echo \"############## ROOTSYS : ##############\"" << endl;
2084 out << "echo $ROOTSYS" << endl;
2085 out << "echo \"############## which root : ##############\"" << endl;
2086 out << "which root" << endl;
2087 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
2088 out << "echo $ALICE_ROOT" << endl;
2089 out << "echo \"############## which aliroot : ##############\"" << endl;
2090 out << "which aliroot" << endl;
2091 out << "echo \"############## system limits : ##############\"" << endl;
2092 out << "ulimit -a" << endl;
2093 out << "echo \"############## memory : ##############\"" << endl;
2094 out << "free -m" << endl;
2095 out << "echo \"=========================================\"" << endl << endl;
2096 // Make sure we can properly compile par files
2097 if (TObject::TestBit(AliAnalysisGrid::kUsePars)) out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
2098 out << fExecutableCommand << " ";
2099 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
2100 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
2101 out << "echo \"############## memory after: ##############\"" << endl;
2102 out << "free -m" << endl;
2103 out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
2104 out << "dmesg | tail -n 10" << endl;
2105 }
2106 Bool_t copy = kTRUE;
2107 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2108 if (copy) {
2109 CdWork();
2110 TString workdir = gGrid->GetHomeDirectory();
2111 TString bindir = Form("%s/bin", workdir.Data());
2112 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir);
2113 workdir += fGridWorkingDir;
2114 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
2115 if (FileExists(executable)) gGrid->Rm(executable);
2116 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
2117 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
2118 }
2119}
2120
2121//______________________________________________________________________________
2122void AliAnalysisAlien::WriteProductionFile(const char *filename) const
2123{
2124// Write the production file to be submitted by LPM manager. The format is:
2125// First line: full_path_to_jdl estimated_no_subjobs_per_master
2126// Next lines: full_path_to_dataset XXX (XXX is a string)
2127// To submit, one has to: submit jdl XXX for all lines
2128 ofstream out;
2129 out.open(filename, ios::out);
2130 if (out.bad()) {
2131 Error("WriteProductionFile", "Bad file name: %s", filename);
2132 return;
2133 }
2134 TString workdir = gGrid->GetHomeDirectory();
2135 workdir += fGridWorkingDir;
2136 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
2137 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
2138 out << locjdl << " " << njobspermaster << endl;
2139 Int_t nmasterjobs = fInputFiles->GetEntries();
2140 for (Int_t i=0; i<nmasterjobs; i++) {
2141 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
2142 }
2143 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
2144 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
2145}
2146
2147//______________________________________________________________________________
2148void AliAnalysisAlien::WriteValidationScript()
2149{
2150// Generate the alien validation script.
2151 // Generate the validation script
2152 TObjString *os;
2153 TString validationScript = fExecutable;
2154 validationScript.ReplaceAll(".sh", "_validation.sh");
2155 if (!Connect()) {
2156 Error("WriteValidationScript", "Alien connection required");
2157 return;
2158 }
2159 TString out_stream = "";
2160 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
2161 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2162 ofstream out;
2163 out.open(validationScript, ios::out);
2164 out << "#!/bin/bash" << endl;
2165 out << "##################################################" << endl;
2166 out << "validateout=`dirname $0`" << endl;
2167 out << "validatetime=`date`" << endl;
2168 out << "validated=\"0\";" << endl;
2169 out << "error=0" << endl;
2170 out << "if [ -z $validateout ]" << endl;
2171 out << "then" << endl;
2172 out << " validateout=\".\"" << endl;
2173 out << "fi" << endl << endl;
2174 out << "cd $validateout;" << endl;
2175 out << "validateworkdir=`pwd`;" << endl << endl;
2176 out << "echo \"*******************************************************\"" << out_stream << endl;
2177 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
2178 out << "" << endl;
2179 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
2180 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
2181 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
2182 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2183 out << "ls -la ./" << out_stream << endl;
2184 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
2185 out << "##################################################" << endl;
2186
2187 out << "" << endl;
2188 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
2189 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
2190 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
2191 out << "" << endl;
2192
2193 out << "if [ ! -f stderr ] ; then" << endl;
2194 out << " error=1" << endl;
2195 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
2196 out << " echo \"Error = $error\" " << out_stream << endl;
2197 out << "fi" << endl;
2198
2199 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
2200 out << " error=1" << endl;
2201 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
2202 out << " echo \"$parArch\" " << out_stream << endl;
2203 out << " echo \"Error = $error\" " << out_stream << endl;
2204 out << "fi" << endl;
2205
2206 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
2207 out << " error=1" << endl;
2208 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
2209 out << " echo \"$segViol\" " << out_stream << endl;
2210 out << " echo \"Error = $error\" " << out_stream << endl;
2211 out << "fi" << endl;
2212
2213 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
2214 out << " error=1" << endl;
2215 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
2216 out << " echo \"$segFault\" " << out_stream << endl;
2217 out << " echo \"Error = $error\" " << out_stream << endl;
2218 out << "fi" << endl;
2219
2220 // Part dedicated to the specific analyses running into the train
2221
2222 TObjArray *arr = fOutputFiles.Tokenize(" ");
2223 TIter next1(arr);
2224 TString output_file;
2225 while ((os=(TObjString*)next1())) {
2226 output_file = os->GetString();
2227 Int_t index = output_file.Index("@");
2228 if (index > 0) output_file.Remove(index);
2229 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
2230 out << " error=1" << endl;
2231 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
2232 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
2233 out << "fi" << endl;
2234 }
2235 delete arr;
2236 out << "if ! [ -f outputs_valid ] ; then" << endl;
2237 out << " error=1" << endl;
2238 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
2239 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
2240 out << "fi" << endl;
2241
2242 out << "if [ $error = 0 ] ; then" << endl;
2243 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
2244 out << "fi" << endl;
2245
2246 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
2247 out << "echo \"*******************************************************\"" << out_stream << endl;
2248 out << "cd -" << endl;
2249 out << "exit $error" << endl;
2250 }
2251 Bool_t copy = kTRUE;
2252 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
2253 if (copy) {
2254 CdWork();
2255 TString workdir = gGrid->GetHomeDirectory();
2256 workdir += fGridWorkingDir;
2257 Info("CreateJDL", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
2258 if (FileExists(validationScript)) gGrid->Rm(validationScript);
2259 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
2260 }
2261}