]> git.uio.no Git - u/mrichter/AliRoot.git/blame_incremental - ANALYSIS/AliAnalysisAlien.cxx
git://git.uio.no / u / mrichter / AliRoot.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
changing default for cut on tracks which have shared TPC clusters to ***OFF*** (was...
[u/mrichter/AliRoot.git] / ANALYSIS / AliAnalysisAlien.cxx
... / ...
CommitLineData
1/**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 * *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
6 * *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
15
16// Author: Mihaela Gheata, 01/09/2008
17
18//==============================================================================
19// AliAnalysisAlien - AliEn utility class. Provides interface for creating
20// a personalized JDL, finding and creating a dataset.
21//==============================================================================
22
23#include "Riostream.h"
24#include "TEnv.h"
25#include "TBits.h"
26#include "TError.h"
27#include "TROOT.h"
28#include "TSystem.h"
29#include "TFile.h"
30#include "TFileCollection.h"
31#include "TChain.h"
32#include "TObjString.h"
33#include "TObjArray.h"
34#include "TGrid.h"
35#include "TGridResult.h"
36#include "TGridCollection.h"
37#include "TGridJDL.h"
38#include "TGridJobStatusList.h"
39#include "TGridJobStatus.h"
40#include "TFileMerger.h"
41#include "AliAnalysisManager.h"
42#include "AliVEventHandler.h"
43#include "AliAnalysisDataContainer.h"
44#include "AliAnalysisAlien.h"
45
46ClassImp(AliAnalysisAlien)
47
48//______________________________________________________________________________
49AliAnalysisAlien::AliAnalysisAlien()
50 :AliAnalysisGrid(),
51 fGridJDL(NULL),
52 fMergingJDL(NULL),
53 fPrice(0),
54 fTTL(0),
55 fSplitMaxInputFileNumber(0),
56 fMaxInitFailed(0),
57 fMasterResubmitThreshold(0),
58 fNtestFiles(0),
59 fNrunsPerMaster(0),
60 fMaxMergeFiles(0),
61 fMaxMergeStages(0),
62 fNsubmitted(0),
63 fProductionMode(0),
64 fOutputToRunNo(0),
65 fMergeViaJDL(0),
66 fFastReadOption(0),
67 fOverwriteMode(1),
68 fNreplicas(2),
69 fNproofWorkers(0),
70 fNproofWorkersPerSlave(0),
71 fProofReset(0),
72 fRunNumbers(),
73 fExecutable(),
74 fExecutableCommand(),
75 fArguments(),
76 fExecutableArgs(),
77 fAnalysisMacro(),
78 fAnalysisSource(),
79 fValidationScript(),
80 fAdditionalRootLibs(),
81 fAdditionalLibs(),
82 fSplitMode(),
83 fAPIVersion(),
84 fROOTVersion(),
85 fAliROOTVersion(),
86 fExternalPackages(),
87 fUser(),
88 fGridWorkingDir(),
89 fGridDataDir(),
90 fDataPattern(),
91 fGridOutputDir(),
92 fOutputArchive(),
93 fOutputFiles(),
94 fInputFormat(),
95 fDatasetName(),
96 fJDLName(),
97 fTerminateFiles(),
98 fMergeExcludes(),
99 fIncludePath(),
100 fCloseSE(),
101 fFriendChainName(),
102 fJobTag(),
103 fOutputSingle(),
104 fRunPrefix(),
105 fProofCluster(),
106 fProofDataSet(),
107 fFileForTestMode(),
108 fRootVersionForProof(),
109 fAliRootMode(),
110 fMergeDirName(),
111 fInputFiles(0),
112 fPackages(0)
113{
114// Dummy ctor.
115 SetDefaults();
116}
117
118//______________________________________________________________________________
119AliAnalysisAlien::AliAnalysisAlien(const char *name)
120 :AliAnalysisGrid(name),
121 fGridJDL(NULL),
122 fMergingJDL(NULL),
123 fPrice(0),
124 fTTL(0),
125 fSplitMaxInputFileNumber(0),
126 fMaxInitFailed(0),
127 fMasterResubmitThreshold(0),
128 fNtestFiles(0),
129 fNrunsPerMaster(0),
130 fMaxMergeFiles(0),
131 fMaxMergeStages(0),
132 fNsubmitted(0),
133 fProductionMode(0),
134 fOutputToRunNo(0),
135 fMergeViaJDL(0),
136 fFastReadOption(0),
137 fOverwriteMode(1),
138 fNreplicas(2),
139 fNproofWorkers(0),
140 fNproofWorkersPerSlave(0),
141 fProofReset(0),
142 fRunNumbers(),
143 fExecutable(),
144 fExecutableCommand(),
145 fArguments(),
146 fExecutableArgs(),
147 fAnalysisMacro(),
148 fAnalysisSource(),
149 fValidationScript(),
150 fAdditionalRootLibs(),
151 fAdditionalLibs(),
152 fSplitMode(),
153 fAPIVersion(),
154 fROOTVersion(),
155 fAliROOTVersion(),
156 fExternalPackages(),
157 fUser(),
158 fGridWorkingDir(),
159 fGridDataDir(),
160 fDataPattern(),
161 fGridOutputDir(),
162 fOutputArchive(),
163 fOutputFiles(),
164 fInputFormat(),
165 fDatasetName(),
166 fJDLName(),
167 fTerminateFiles(),
168 fMergeExcludes(),
169 fIncludePath(),
170 fCloseSE(),
171 fFriendChainName(),
172 fJobTag(),
173 fOutputSingle(),
174 fRunPrefix(),
175 fProofCluster(),
176 fProofDataSet(),
177 fFileForTestMode(),
178 fRootVersionForProof(),
179 fAliRootMode(),
180 fMergeDirName(),
181 fInputFiles(0),
182 fPackages(0)
183{
184// Default ctor.
185 SetDefaults();
186}
187
188//______________________________________________________________________________
189AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
190 :AliAnalysisGrid(other),
191 fGridJDL(NULL),
192 fMergingJDL(NULL),
193 fPrice(other.fPrice),
194 fTTL(other.fTTL),
195 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
196 fMaxInitFailed(other.fMaxInitFailed),
197 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
198 fNtestFiles(other.fNtestFiles),
199 fNrunsPerMaster(other.fNrunsPerMaster),
200 fMaxMergeFiles(other.fMaxMergeFiles),
201 fMaxMergeStages(other.fMaxMergeStages),
202 fNsubmitted(other.fNsubmitted),
203 fProductionMode(other.fProductionMode),
204 fOutputToRunNo(other.fOutputToRunNo),
205 fMergeViaJDL(other.fMergeViaJDL),
206 fFastReadOption(other.fFastReadOption),
207 fOverwriteMode(other.fOverwriteMode),
208 fNreplicas(other.fNreplicas),
209 fNproofWorkers(other.fNproofWorkers),
210 fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
211 fProofReset(other.fProofReset),
212 fRunNumbers(other.fRunNumbers),
213 fExecutable(other.fExecutable),
214 fExecutableCommand(other.fExecutableCommand),
215 fArguments(other.fArguments),
216 fExecutableArgs(other.fExecutableArgs),
217 fAnalysisMacro(other.fAnalysisMacro),
218 fAnalysisSource(other.fAnalysisSource),
219 fValidationScript(other.fValidationScript),
220 fAdditionalRootLibs(other.fAdditionalRootLibs),
221 fAdditionalLibs(other.fAdditionalLibs),
222 fSplitMode(other.fSplitMode),
223 fAPIVersion(other.fAPIVersion),
224 fROOTVersion(other.fROOTVersion),
225 fAliROOTVersion(other.fAliROOTVersion),
226 fExternalPackages(other.fExternalPackages),
227 fUser(other.fUser),
228 fGridWorkingDir(other.fGridWorkingDir),
229 fGridDataDir(other.fGridDataDir),
230 fDataPattern(other.fDataPattern),
231 fGridOutputDir(other.fGridOutputDir),
232 fOutputArchive(other.fOutputArchive),
233 fOutputFiles(other.fOutputFiles),
234 fInputFormat(other.fInputFormat),
235 fDatasetName(other.fDatasetName),
236 fJDLName(other.fJDLName),
237 fTerminateFiles(other.fTerminateFiles),
238 fMergeExcludes(other.fMergeExcludes),
239 fIncludePath(other.fIncludePath),
240 fCloseSE(other.fCloseSE),
241 fFriendChainName(other.fFriendChainName),
242 fJobTag(other.fJobTag),
243 fOutputSingle(other.fOutputSingle),
244 fRunPrefix(other.fRunPrefix),
245 fProofCluster(other.fProofCluster),
246 fProofDataSet(other.fProofDataSet),
247 fFileForTestMode(other.fFileForTestMode),
248 fRootVersionForProof(other.fRootVersionForProof),
249 fAliRootMode(other.fAliRootMode),
250 fMergeDirName(other.fMergeDirName),
251 fInputFiles(0),
252 fPackages(0)
253{
254// Copy ctor.
255 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
256 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
257 fRunRange[0] = other.fRunRange[0];
258 fRunRange[1] = other.fRunRange[1];
259 if (other.fInputFiles) {
260 fInputFiles = new TObjArray();
261 TIter next(other.fInputFiles);
262 TObject *obj;
263 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
264 fInputFiles->SetOwner();
265 }
266 if (other.fPackages) {
267 fPackages = new TObjArray();
268 TIter next(other.fPackages);
269 TObject *obj;
270 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
271 fPackages->SetOwner();
272 }
273}
274
275//______________________________________________________________________________
276AliAnalysisAlien::~AliAnalysisAlien()
277{
278// Destructor.
279 if (fGridJDL) delete fGridJDL;
280 if (fMergingJDL) delete fMergingJDL;
281 if (fInputFiles) delete fInputFiles;
282 if (fPackages) delete fPackages;
283}
284
285//______________________________________________________________________________
286AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
287{
288// Assignment.
289 if (this != &other) {
290 AliAnalysisGrid::operator=(other);
291 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
292 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
293 fPrice = other.fPrice;
294 fTTL = other.fTTL;
295 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
296 fMaxInitFailed = other.fMaxInitFailed;
297 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
298 fNtestFiles = other.fNtestFiles;
299 fNrunsPerMaster = other.fNrunsPerMaster;
300 fMaxMergeFiles = other.fMaxMergeFiles;
301 fMaxMergeStages = other.fMaxMergeStages;
302 fNsubmitted = other.fNsubmitted;
303 fProductionMode = other.fProductionMode;
304 fOutputToRunNo = other.fOutputToRunNo;
305 fMergeViaJDL = other.fMergeViaJDL;
306 fFastReadOption = other.fFastReadOption;
307 fOverwriteMode = other.fOverwriteMode;
308 fNreplicas = other.fNreplicas;
309 fNproofWorkers = other.fNproofWorkers;
310 fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
311 fProofReset = other.fProofReset;
312 fRunNumbers = other.fRunNumbers;
313 fExecutable = other.fExecutable;
314 fExecutableCommand = other.fExecutableCommand;
315 fArguments = other.fArguments;
316 fExecutableArgs = other.fExecutableArgs;
317 fAnalysisMacro = other.fAnalysisMacro;
318 fAnalysisSource = other.fAnalysisSource;
319 fValidationScript = other.fValidationScript;
320 fAdditionalRootLibs = other.fAdditionalRootLibs;
321 fAdditionalLibs = other.fAdditionalLibs;
322 fSplitMode = other.fSplitMode;
323 fAPIVersion = other.fAPIVersion;
324 fROOTVersion = other.fROOTVersion;
325 fAliROOTVersion = other.fAliROOTVersion;
326 fExternalPackages = other.fExternalPackages;
327 fUser = other.fUser;
328 fGridWorkingDir = other.fGridWorkingDir;
329 fGridDataDir = other.fGridDataDir;
330 fDataPattern = other.fDataPattern;
331 fGridOutputDir = other.fGridOutputDir;
332 fOutputArchive = other.fOutputArchive;
333 fOutputFiles = other.fOutputFiles;
334 fInputFormat = other.fInputFormat;
335 fDatasetName = other.fDatasetName;
336 fJDLName = other.fJDLName;
337 fTerminateFiles = other.fTerminateFiles;
338 fMergeExcludes = other.fMergeExcludes;
339 fIncludePath = other.fIncludePath;
340 fCloseSE = other.fCloseSE;
341 fFriendChainName = other.fFriendChainName;
342 fJobTag = other.fJobTag;
343 fOutputSingle = other.fOutputSingle;
344 fRunPrefix = other.fRunPrefix;
345 fProofCluster = other.fProofCluster;
346 fProofDataSet = other.fProofDataSet;
347 fFileForTestMode = other.fFileForTestMode;
348 fRootVersionForProof = other.fRootVersionForProof;
349 fAliRootMode = other.fAliRootMode;
350 fMergeDirName = other.fMergeDirName;
351 if (other.fInputFiles) {
352 fInputFiles = new TObjArray();
353 TIter next(other.fInputFiles);
354 TObject *obj;
355 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
356 fInputFiles->SetOwner();
357 }
358 if (other.fPackages) {
359 fPackages = new TObjArray();
360 TIter next(other.fPackages);
361 TObject *obj;
362 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
363 fPackages->SetOwner();
364 }
365 }
366 return *this;
367}
368
369//______________________________________________________________________________
370void AliAnalysisAlien::SetRunPrefix(const char *prefix)
371{
372// Set the run number format. Can be a prefix or a format like "%09d"
373 fRunPrefix = prefix;
374 if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
375}
376
377//______________________________________________________________________________
378void AliAnalysisAlien::AddIncludePath(const char *path)
379{
380// Add include path in the remote analysis macro.
381 TString p(path);
382 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
383 else fIncludePath += Form("-I%s ", path);
384}
385
386//______________________________________________________________________________
387void AliAnalysisAlien::AddRunNumber(Int_t run)
388{
389// Add a run number to the list of runs to be processed.
390 if (fRunNumbers.Length()) fRunNumbers += " ";
391 fRunNumbers += Form(fRunPrefix.Data(), run);
392}
393
394//______________________________________________________________________________
395void AliAnalysisAlien::AddRunList(const char* runList)
396{
397// Add several runs into the list of runs; they are expected to be separated by a blank character.
398 TString sList = runList;
399 TObjArray *list = sList.Tokenize(" ");
400 Int_t n = list->GetEntries();
401 for (Int_t i = 0; i < n; i++) {
402 TObjString *os = (TObjString*)list->At(i);
403 AddRunNumber(os->GetString().Atoi());
404 }
405 delete list;
406}
407
408//______________________________________________________________________________
409void AliAnalysisAlien::AddRunNumber(const char* run)
410{
411// Add a run number to the list of runs to be processed.
412 if (fRunNumbers.Length()) fRunNumbers += " ";
413 fRunNumbers += run;
414}
415
416//______________________________________________________________________________
417void AliAnalysisAlien::AddDataFile(const char *lfn)
418{
419// Adds a data file to the input to be analysed. The file should be a valid LFN
420// or point to an existing file in the alien workdir.
421 if (!fInputFiles) fInputFiles = new TObjArray();
422 fInputFiles->Add(new TObjString(lfn));
423}
424
425//______________________________________________________________________________
426void AliAnalysisAlien::AddExternalPackage(const char *package)
427{
428// Adds external packages w.r.t to the default ones (root,aliroot and gapi)
429 if (fExternalPackages) fExternalPackages += " ";
430 fExternalPackages += package;
431}
432
433//______________________________________________________________________________
434Bool_t AliAnalysisAlien::Connect()
435{
436// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
437 if (gGrid && gGrid->IsConnected()) return kTRUE;
438 if (fProductionMode) return kTRUE;
439 if (!gGrid) {
440 Info("Connect", "Trying to connect to AliEn ...");
441 TGrid::Connect("alien://");
442 }
443 if (!gGrid || !gGrid->IsConnected()) {
444 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
445 return kFALSE;
446 }
447 fUser = gGrid->GetUser();
448 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
449 return kTRUE;
450}
451
452//______________________________________________________________________________
453void AliAnalysisAlien::CdWork()
454{
455// Check validity of alien workspace. Create directory if possible.
456 if (!Connect()) {
457 Error("CdWork", "Alien connection required");
458 return;
459 }
460 TString homedir = gGrid->GetHomeDirectory();
461 TString workdir = homedir + fGridWorkingDir;
462 if (DirectoryExists(workdir)) {
463 gGrid->Cd(workdir);
464 return;
465 }
466 // Work directory not existing - create it
467 gGrid->Cd(homedir);
468 if (gGrid->Mkdir(workdir, "-p")) {
469 gGrid->Cd(fGridWorkingDir);
470 Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
471 } else {
472 Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
473 workdir.Data(), homedir.Data());
474 fGridWorkingDir = "";
475 }
476}
477
478//______________________________________________________________________________
479Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
480{
481// Check if file copying is possible.
482 if (fProductionMode) return kTRUE;
483 if (!Connect()) {
484 Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
485 return kFALSE;
486 }
487 Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
488 \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
489 // Check if alien_CLOSE_SE is defined
490 TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
491 if (!closeSE.IsNull()) {
492 Info("CheckFileCopy", "Your current close storage is pointing to: \
493 \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
494 } else {
495 Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
496 }
497 // Check if grid directory exists.
498 if (!DirectoryExists(alienpath)) {
499 Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
500 return kFALSE;
501 }
502 TFile f("plugin_test_copy", "RECREATE");
503 // User may not have write permissions to current directory
504 if (f.IsZombie()) {
505 Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
506 gSystem->WorkingDirectory());
507 return kFALSE;
508 }
509 f.Close();
510 if (FileExists(Form("alien://%s/%s",alienpath, f.GetName()))) gGrid->Rm(Form("alien://%s/%s",alienpath, f.GetName()));
511 if (!TFile::Cp(f.GetName(), Form("alien://%s/%s",alienpath, f.GetName()))) {
512 Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
513 \n# 1. Make sure you have write permissions there. If this is the case: \
514 \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
515 \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
516 \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
517 \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
518 gSystem->Unlink(f.GetName());
519 return kFALSE;
520 }
521 gSystem->Unlink(f.GetName());
522 gGrid->Rm(Form("%s%s",alienpath,f.GetName()));
523 Info("CheckFileCopy", "### ...SUCCESS ###");
524 return kTRUE;
525}
526
527//______________________________________________________________________________
528Bool_t AliAnalysisAlien::CheckInputData()
529{
530// Check validity of input data. If necessary, create xml files.
531 if (fProductionMode) return kTRUE;
532 if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
533 if (!fGridDataDir.Length()) {
534 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
535 return kFALSE;
536 }
537 if (fMergeViaJDL) {
538 Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
539 return kFALSE;
540 }
541 Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
542 if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
543 TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
544 return kTRUE;
545 }
546 // Process declared files
547 Bool_t isCollection = kFALSE;
548 Bool_t isXml = kFALSE;
549 Bool_t useTags = kFALSE;
550 Bool_t checked = kFALSE;
551 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
552 TString file;
553 TString workdir = gGrid->GetHomeDirectory();
554 workdir += fGridWorkingDir;
555 if (fInputFiles) {
556 TObjString *objstr;
557 TIter next(fInputFiles);
558 while ((objstr=(TObjString*)next())) {
559 file = workdir;
560 file += "/";
561 file += objstr->GetString();
562 // Store full lfn path
563 if (FileExists(file)) objstr->SetString(file);
564 else {
565 file = objstr->GetName();
566 if (!FileExists(objstr->GetName())) {
567 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
568 objstr->GetName(), workdir.Data());
569 return kFALSE;
570 }
571 }
572 Bool_t iscoll, isxml, usetags;
573 CheckDataType(file, iscoll, isxml, usetags);
574 if (!checked) {
575 checked = kTRUE;
576 isCollection = iscoll;
577 isXml = isxml;
578 useTags = usetags;
579 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
580 } else {
581 if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
582 Error("CheckInputData", "Some conflict was found in the types of inputs");
583 return kFALSE;
584 }
585 }
586 }
587 }
588 // Process requested run numbers
589 if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
590 // Check validity of alien data directory
591 if (!fGridDataDir.Length()) {
592 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
593 return kFALSE;
594 }
595 if (!DirectoryExists(fGridDataDir)) {
596 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
597 return kFALSE;
598 }
599 if (isCollection) {
600 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
601 return kFALSE;
602 }
603
604 if (checked && !isXml) {
605 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
606 return kFALSE;
607 }
608 // Check validity of run number(s)
609 TObjArray *arr;
610 TObjString *os;
611 TString format;
612 Int_t nruns = 0;
613 TString schunk, schunk2;
614 TString path;
615 if (!checked) {
616 checked = kTRUE;
617 useTags = fDataPattern.Contains("tag");
618 TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
619 }
620 if (useTags != fDataPattern.Contains("tag")) {
621 Error("CheckInputData", "Cannot mix input files using/not using tags");
622 return kFALSE;
623 }
624 if (fRunNumbers.Length()) {
625 Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
626 arr = fRunNumbers.Tokenize(" ");
627 TIter next(arr);
628 while ((os=(TObjString*)next())) {
629 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
630 if (!DirectoryExists(path)) {
631 Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
632 continue;
633 }
634 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
635 TString msg = "\n##### file: ";
636 msg += path;
637 msg += " type: xml_collection;";
638 if (useTags) msg += " using_tags: Yes";
639 else msg += " using_tags: No";
640 Info("CheckDataType", "%s", msg.Data());
641 if (fNrunsPerMaster<2) {
642 AddDataFile(Form("%s.xml", os->GetString().Data()));
643 } else {
644 nruns++;
645 if (((nruns-1)%fNrunsPerMaster) == 0) {
646 schunk = os->GetString();
647 }
648 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
649 schunk += Form("_%s.xml", os->GetString().Data());
650 AddDataFile(schunk);
651 }
652 }
653 delete arr;
654 } else {
655 Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
656 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
657 format = Form("%%s/%s ", fRunPrefix.Data());
658 path = Form(format.Data(), fGridDataDir.Data(), irun);
659 if (!DirectoryExists(path)) {
660 continue;
661 }
662 format = Form("%%s/%s.xml", fRunPrefix.Data());
663 path = Form(format.Data(), workdir.Data(),irun);
664 TString msg = "\n##### file: ";
665 msg += path;
666 msg += " type: xml_collection;";
667 if (useTags) msg += " using_tags: Yes";
668 else msg += " using_tags: No";
669 Info("CheckDataType", "%s", msg.Data());
670 if (fNrunsPerMaster<2) {
671 format = Form("%s.xml", fRunPrefix.Data());
672 AddDataFile(Form(format.Data(),irun));
673 } else {
674 nruns++;
675 if (((nruns-1)%fNrunsPerMaster) == 0) {
676 schunk = Form(fRunPrefix.Data(),irun);
677 }
678 format = Form("_%s.xml", fRunPrefix.Data());
679 schunk2 = Form(format.Data(), irun);
680 if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
681 schunk += schunk2;
682 AddDataFile(schunk);
683 }
684 }
685 if (!fInputFiles) {
686 schunk += schunk2;
687 AddDataFile(schunk);
688 }
689 }
690 return kTRUE;
691}
692
693//______________________________________________________________________________
694Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
695{
696// Create dataset for the grid data directory + run number.
697 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
698 if (!Connect()) {
699 Error("CreateDataset", "Cannot create dataset with no grid connection");
700 return kFALSE;
701 }
702
703 // Cd workspace
704 if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
705 TString workdir = gGrid->GetHomeDirectory();
706 workdir += fGridWorkingDir;
707
708 // Compose the 'find' command arguments
709 TString format;
710 TString command;
711 TString options = "-x collection ";
712 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
713 TString conditions = "";
714
715 TString file;
716 TString path;
717 Int_t nruns = 0;
718 TString schunk, schunk2;
719 TGridCollection *cbase=0, *cadd=0;
720 if (!fRunNumbers.Length() && !fRunRange[0]) {
721 if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
722 // Make a single data collection from data directory.
723 path = fGridDataDir;
724 if (!DirectoryExists(path)) {
725 Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
726 return kFALSE;
727 }
728// CdWork();
729 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
730 else file = Form("%s.xml", gSystem->BaseName(path));
731 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
732 command = "find ";
733 command += options;
734 command += path;
735 command += " ";
736 command += pattern;
737 command += conditions;
738 printf("command: %s\n", command.Data());
739 TGridResult *res = gGrid->Command(command);
740 if (res) delete res;
741 // Write standard output to file
742 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
743 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
744 Bool_t nullFile = kFALSE;
745 if (!hasGrep) {
746 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
747 } else {
748 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
749 if (nullFile) {
750 Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
751 return kFALSE;
752 }
753 }
754 }
755 Bool_t fileExists = FileExists(file);
756 if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
757 // Copy xml file to alien space
758 if (fileExists) gGrid->Rm(file);
759 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
760 if (!FileExists(file)) {
761 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
762 return kFALSE;
763 }
764 // Update list of files to be processed.
765 }
766 AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
767 return kTRUE;
768 }
769 // Several runs
770 Bool_t nullResult = kTRUE;
771 if (fRunNumbers.Length()) {
772 TObjArray *arr = fRunNumbers.Tokenize(" ");
773 TObjString *os;
774 TIter next(arr);
775 while ((os=(TObjString*)next())) {
776 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
777 if (!DirectoryExists(path)) continue;
778// CdWork();
779 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
780 else file = Form("%s.xml", os->GetString().Data());
781 // If local collection file does not exist, create it via 'find' command.
782 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
783 command = "find ";
784 command += options;
785 command += path;
786 command += pattern;
787 command += conditions;
788 TGridResult *res = gGrid->Command(command);
789 if (res) delete res;
790 // Write standard output to file
791 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
792 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
793 Bool_t nullFile = kFALSE;
794 if (!hasGrep) {
795 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
796 } else {
797 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
798 if (nullFile) {
799 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
800 fRunNumbers.ReplaceAll(os->GetString().Data(), "");
801 continue;
802 }
803 }
804 nullResult = kFALSE;
805 }
806 if (TestBit(AliAnalysisGrid::kTest)) break;
807 // Check if there is one run per master job.
808 if (fNrunsPerMaster<2) {
809 if (FileExists(file)) {
810 if (fOverwriteMode) gGrid->Rm(file);
811 else {
812 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
813 continue;
814 }
815 }
816 // Copy xml file to alien space
817 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
818 if (!FileExists(file)) {
819 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
820 delete arr;
821 return kFALSE;
822 }
823 } else {
824 nruns++;
825 if (((nruns-1)%fNrunsPerMaster) == 0) {
826 schunk = os->GetString();
827 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
828 } else {
829 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
830 printf(" Merging collection <%s> into masterjob input...\n", file.Data());
831 cbase->Add(cadd);
832 delete cadd;
833 }
834 if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
835 continue;
836 }
837 schunk += Form("_%s.xml", os->GetString().Data());
838 if (FileExists(schunk)) {
839 if (fOverwriteMode) gGrid->Rm(file);
840 else {
841 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
842 continue;
843 }
844 }
845 printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
846 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
847 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
848 if (!FileExists(schunk)) {
849 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
850 delete arr;
851 return kFALSE;
852 }
853 }
854 }
855 delete arr;
856 if (nullResult) {
857 Error("CreateDataset", "No valid dataset corresponding to the query!");
858 return kFALSE;
859 }
860 } else {
861 // Process a full run range.
862 for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
863 format = Form("%%s/%s ", fRunPrefix.Data());
864 path = Form(format.Data(), fGridDataDir.Data(), irun);
865 if (!DirectoryExists(path)) continue;
866// CdWork();
867 format = Form("%s.xml", fRunPrefix.Data());
868 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
869 else file = Form(format.Data(), irun);
870 if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
871 if (fOverwriteMode) gGrid->Rm(file);
872 else {
873 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
874 continue;
875 }
876 }
877 // If local collection file does not exist, create it via 'find' command.
878 if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
879 command = "find ";
880 command += options;
881 command += path;
882 command += pattern;
883 command += conditions;
884 TGridResult *res = gGrid->Command(command);
885 if (res) delete res;
886 // Write standard output to file
887 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
888 Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
889 Bool_t nullFile = kFALSE;
890 if (!hasGrep) {
891 Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
892 } else {
893 nullFile = (gSystem->Exec(Form("grep /event %s 2>/dev/null > /dev/null",file.Data()))==0)?kFALSE:kTRUE;
894 if (nullFile) {
895 Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
896 continue;
897 }
898 }
899 nullResult = kFALSE;
900 }
901 if (TestBit(AliAnalysisGrid::kTest)) break;
902 // Check if there is one run per master job.
903 if (fNrunsPerMaster<2) {
904 if (FileExists(file)) {
905 if (fOverwriteMode) gGrid->Rm(file);
906 else {
907 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
908 continue;
909 }
910 }
911 // Copy xml file to alien space
912 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
913 if (!FileExists(file)) {
914 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
915 return kFALSE;
916 }
917 } else {
918 nruns++;
919 // Check if the collection for the chunk exist locally.
920 Int_t nchunk = (nruns-1)/fNrunsPerMaster;
921 if (FileExists(fInputFiles->At(nchunk)->GetName())) {
922 if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
923 else continue;
924 }
925 printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
926 if (((nruns-1)%fNrunsPerMaster) == 0) {
927 schunk = Form(fRunPrefix.Data(), irun);
928 cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
929 } else {
930 cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
931 cbase->Add(cadd);
932 delete cadd;
933 }
934 format = Form("%%s_%s.xml", fRunPrefix.Data());
935 schunk2 = Form(format.Data(), schunk.Data(), irun);
936 if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
937 continue;
938 }
939 schunk = schunk2;
940 if (FileExists(schunk)) {
941 if (fOverwriteMode) gGrid->Rm(schunk);
942 else {
943 Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
944 continue;
945 }
946 }
947 printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
948 cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
949 if (FileExists(schunk)) {
950 if (fOverwriteMode) gGrid->Rm(schunk);
951 else {
952 Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
953 continue;
954 }
955 }
956 TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
957 if (!FileExists(schunk)) {
958 Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
959 return kFALSE;
960 }
961 }
962 }
963 if (nullResult) {
964 Error("CreateDataset", "No valid dataset corresponding to the query!");
965 return kFALSE;
966 }
967 }
968 return kTRUE;
969}
970
971//______________________________________________________________________________
972Bool_t AliAnalysisAlien::CreateJDL()
973{
974// Generate a JDL file according to current settings. The name of the file is
975// specified by fJDLName.
976 Bool_t error = kFALSE;
977 TObjArray *arr = 0;
978 Bool_t copy = kTRUE;
979 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
980 Bool_t generate = kTRUE;
981 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
982 if (!Connect()) {
983 Error("CreateJDL", "Alien connection required");
984 return kFALSE;
985 }
986 // Check validity of alien workspace
987 TString workdir;
988 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
989 if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
990 workdir += fGridWorkingDir;
991 if (generate) {
992 TObjString *os;
993 if (!fInputFiles) {
994 Error("CreateJDL()", "Define some input files for your analysis.");
995 error = kTRUE;
996 }
997 // Compose list of input files
998 // Check if output files were defined
999 if (!fOutputFiles.Length()) {
1000 Error("CreateJDL", "You must define at least one output file");
1001 error = kTRUE;
1002 }
1003 // Check if an output directory was defined and valid
1004 if (!fGridOutputDir.Length()) {
1005 Error("CreateJDL", "You must define AliEn output directory");
1006 error = kTRUE;
1007 } else {
1008 if (!fProductionMode) {
1009 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1010 if (!DirectoryExists(fGridOutputDir)) {
1011 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1012 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1013 } else {
1014 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1015 // error = kTRUE;
1016 }
1017 } else {
1018 Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
1019 }
1020 gGrid->Cd(workdir);
1021 }
1022 }
1023 // Exit if any error up to now
1024 if (error) return kFALSE;
1025 // Set JDL fields
1026 if (!fUser.IsNull()) {
1027 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1028 fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1029 }
1030 fGridJDL->SetExecutable(fExecutable, "This is the startup script");
1031 TString mergeExec = fExecutable;
1032 mergeExec.ReplaceAll(".sh", "_merge.sh");
1033 fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1034 mergeExec.ReplaceAll(".sh", ".C");
1035 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),mergeExec.Data()), "List of input files to be uploaded to workers");
1036 if (!fArguments.IsNull())
1037 fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
1038 if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
1039 else {
1040 if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1041 else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1042 }
1043
1044 fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1045 fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1046 fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1047 fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1048
1049 if (fMaxInitFailed > 0) {
1050 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
1051 fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1052 }
1053 if (fSplitMaxInputFileNumber > 0) {
1054 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
1055 fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
1056 }
1057 if (!IsOneStageMerging()) {
1058 fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
1059 fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
1060 }
1061 if (fSplitMode.Length()) {
1062 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
1063 fGridJDL->SetDescription("Split", "We split per SE or file");
1064 }
1065 fMergingJDL->SetValue("Split", "\"se\"");
1066 fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
1067 if (!fAliROOTVersion.IsNull()) {
1068 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1069 fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1070 }
1071 if (!fROOTVersion.IsNull()) {
1072 fGridJDL->AddToPackages("ROOT", fROOTVersion);
1073 fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1074 }
1075 if (!fAPIVersion.IsNull()) {
1076 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
1077 fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1078 }
1079 if (!fExternalPackages.IsNull()) {
1080 arr = fExternalPackages.Tokenize(" ");
1081 TIter next(arr);
1082 while ((os=(TObjString*)next())) {
1083 TString pkgname = os->GetString();
1084 Int_t index = pkgname.Index("::");
1085 TString pkgversion = pkgname(index+2, pkgname.Length());
1086 pkgname.Remove(index);
1087 fGridJDL->AddToPackages(pkgname, pkgversion);
1088 fMergingJDL->AddToPackages(pkgname, pkgversion);
1089 }
1090 delete arr;
1091 }
1092 fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1093 fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1094 fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1095 fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1096 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
1097 TString analysisFile = fExecutable;
1098 analysisFile.ReplaceAll(".sh", ".root");
1099 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
1100 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
1101 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
1102 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
1103 if (fAdditionalLibs.Length()) {
1104 arr = fAdditionalLibs.Tokenize(" ");
1105 TIter next(arr);
1106 while ((os=(TObjString*)next())) {
1107 if (os->GetString().Contains(".so")) continue;
1108 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
1109 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
1110 }
1111 delete arr;
1112 }
1113 if (fPackages) {
1114 TIter next(fPackages);
1115 TObject *obj;
1116 while ((obj=next())) {
1117 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1118 fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1119 }
1120 }
1121 if (fOutputArchive.Length()) {
1122 arr = fOutputArchive.Tokenize(" ");
1123 TIter next(arr);
1124 Bool_t first = kTRUE;
1125 const char *comment = "Files to be archived";
1126 const char *comment1 = comment;
1127 while ((os=(TObjString*)next())) {
1128 if (!first) comment = NULL;
1129 if (!os->GetString().Contains("@") && fCloseSE.Length())
1130 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1131 else
1132 fGridJDL->AddToOutputArchive(os->GetString(), comment);
1133 first = kFALSE;
1134 }
1135 delete arr;
1136 // Output archive for the merging jdl
1137 TString outputArchive;
1138 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1139 outputArchive = "log_archive.zip:std*@disk=1 ";
1140 // Add normal output files, extra files + terminate files
1141 TString files = GetListOfFiles("outextter");
1142 // Do not register merge excludes
1143 if (!fMergeExcludes.IsNull()) {
1144 arr = fMergeExcludes.Tokenize(" ");
1145 TIter next1(arr);
1146 while ((os=(TObjString*)next1())) {
1147 files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1148 files.ReplaceAll(os->GetString(),"");
1149 }
1150 delete arr;
1151 }
1152 files.ReplaceAll(".root", "*.root");
1153 outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
1154 } else {
1155 TString files = fOutputArchive;
1156 files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1157 outputArchive = files;
1158 }
1159 arr = outputArchive.Tokenize(" ");
1160 TIter next2(arr);
1161 comment = comment1;
1162 first = kTRUE;
1163 while ((os=(TObjString*)next2())) {
1164 if (!first) comment = NULL;
1165 TString currentfile = os->GetString();
1166 if (!currentfile.Contains("@") && fCloseSE.Length())
1167 fMergingJDL->AddToOutputArchive(Form("%s@%s",currentfile.Data(), fCloseSE.Data()), comment);
1168 else
1169 fMergingJDL->AddToOutputArchive(currentfile, comment);
1170 first = kFALSE;
1171 }
1172 delete arr;
1173 }
1174 arr = fOutputFiles.Tokenize(",");
1175 TIter next(arr);
1176 Bool_t first = kTRUE;
1177 const char *comment = "Files to be saved";
1178 while ((os=(TObjString*)next())) {
1179 // Ignore ouputs in jdl that are also in outputarchive
1180 TString sout = os->GetString();
1181 sout.ReplaceAll("*", "");
1182 sout.ReplaceAll(".root", "");
1183 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1184 if (fOutputArchive.Contains(sout)) continue;
1185 if (!first) comment = NULL;
1186 if (!os->GetString().Contains("@") && fCloseSE.Length())
1187 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1188 else
1189 fGridJDL->AddToOutputSandbox(os->GetString(), comment);
1190 first = kFALSE;
1191 if (fMergeExcludes.Contains(sout)) continue;
1192 if (!os->GetString().Contains("@") && fCloseSE.Length())
1193 fMergingJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()), comment);
1194 else
1195 fMergingJDL->AddToOutputSandbox(os->GetString(), comment);
1196 }
1197 delete arr;
1198 fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1199 fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1200 TString validationScript = fValidationScript;
1201 fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1202 validationScript.ReplaceAll(".sh", "_merge.sh");
1203 fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1204 if (fMasterResubmitThreshold) {
1205 fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1206 fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1207 }
1208 // Write a jdl with 2 input parameters: collection name and output dir name.
1209 WriteJDL(copy);
1210 }
1211 // Copy jdl to grid workspace
1212 if (copy) {
1213 // Check if an output directory was defined and valid
1214 if (!fGridOutputDir.Length()) {
1215 Error("CreateJDL", "You must define AliEn output directory");
1216 return kFALSE;
1217 } else {
1218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1219 if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
1220 if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1221 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1222 } else {
1223 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1224 return kFALSE;
1225 }
1226 }
1227 gGrid->Cd(workdir);
1228 }
1229 if (TestBit(AliAnalysisGrid::kSubmit)) {
1230 TString mergeJDLName = fExecutable;
1231 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1232 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
1233 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1234 if (fProductionMode) {
1235 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
1236 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1237 }
1238 if (FileExists(locjdl)) gGrid->Rm(locjdl);
1239 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1240 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1241 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1242 if (fMergeViaJDL) {
1243 Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1244 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1245 }
1246 }
1247 if (fAdditionalLibs.Length()) {
1248 arr = fAdditionalLibs.Tokenize(" ");
1249 TObjString *os;
1250 TIter next(arr);
1251 while ((os=(TObjString*)next())) {
1252 if (os->GetString().Contains(".so")) continue;
1253 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
1254 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1255 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1256 }
1257 delete arr;
1258 }
1259 if (fPackages) {
1260 TIter next(fPackages);
1261 TObject *obj;
1262 while ((obj=next())) {
1263 if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
1264 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1265 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1266 }
1267 }
1268 }
1269 return kTRUE;
1270}
1271
1272//______________________________________________________________________________
1273Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
1274{
1275// Writes one or more JDL's corresponding to findex. If findex is negative,
1276// all run numbers are considered in one go (jdl). For non-negative indices
1277// they correspond to the indices in the array fInputFiles.
1278 if (!fInputFiles) return kFALSE;
1279 TObject *os;
1280 TString workdir;
1281 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1282 workdir += fGridWorkingDir;
1283 TString stageName = "$2";
1284 if (fProductionMode) stageName = "$4";
1285 if (!fMergeDirName.IsNull()) {
1286 fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
1287 fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
1288 } else {
1289 fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
1290 fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
1291 }
1292 if (fProductionMode) {
1293 TIter next(fInputFiles);
1294 while ((os=next())) {
1295 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1296 }
1297 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
1298 } else {
1299 if (!fRunNumbers.Length() && !fRunRange[0]) {
1300 // One jdl with no parameters in case input data is specified by name.
1301 TIter next(fInputFiles);
1302 while ((os=next()))
1303 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
1304 if (!fOutputSingle.IsNull())
1305 fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
1306 else {
1307 fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1308 fMergingJDL->SetOutputDirectory(fGridOutputDir);
1309 }
1310 } else {
1311 // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
1312 fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
1313 if (!fOutputSingle.IsNull()) {
1314 if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
1315 else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
1316 } else {
1317 fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
1318 }
1319 }
1320 }
1321
1322 // Generate the JDL as a string
1323 TString sjdl = fGridJDL->Generate();
1324 TString sjdl1 = fMergingJDL->Generate();
1325 // Final merge jdl
1326 if (!fMergeDirName.IsNull()) {
1327 fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
1328 fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
1329 } else {
1330 fMergingJDL->SetOutputDirectory("$1", "Output directory");
1331 fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
1332 }
1333 TString sjdl2 = fMergingJDL->Generate();
1334 Int_t index, index1;
1335 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
1336 sjdl.ReplaceAll("(member", "\n (member");
1337 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1338 sjdl.ReplaceAll("{", "{\n ");
1339 sjdl.ReplaceAll("};", "\n};");
1340 sjdl.ReplaceAll("{\n \n", "{\n");
1341 sjdl.ReplaceAll("\n\n", "\n");
1342 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
1343 sjdl1.ReplaceAll("\"LF:", "\n \"LF:");
1344 sjdl1.ReplaceAll("(member", "\n (member");
1345 sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1346 sjdl1.ReplaceAll("{", "{\n ");
1347 sjdl1.ReplaceAll("};", "\n};");
1348 sjdl1.ReplaceAll("{\n \n", "{\n");
1349 sjdl1.ReplaceAll("\n\n", "\n");
1350 sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
1351 sjdl2.ReplaceAll("\"LF:", "\n \"LF:");
1352 sjdl2.ReplaceAll("(member", "\n (member");
1353 sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
1354 sjdl2.ReplaceAll("{", "{\n ");
1355 sjdl2.ReplaceAll("};", "\n};");
1356 sjdl2.ReplaceAll("{\n \n", "{\n");
1357 sjdl2.ReplaceAll("\n\n", "\n");
1358 sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
1359 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1360 sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
1361 index = sjdl.Index("JDLVariables");
1362 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
1363 sjdl += "Workdirectorysize = {\"5000MB\"};";
1364 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
1365 sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
1366 index = fJobTag.Index(":");
1367 if (index < 0) index = fJobTag.Length();
1368 TString jobTag = fJobTag;
1369 if (fProductionMode) jobTag.Insert(index,"_Stage$4");
1370 sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
1371 if (fProductionMode) {
1372 sjdl1.Prepend("# Generated merging jdl (production mode) \
1373 \n# $1 = full alien path to output directory to be merged \
1374 \n# $2 = train number \
1375 \n# $3 = production (like LHC10b) \
1376 \n# $4 = merging stage \
1377 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1378 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1379 sjdl2.Prepend("# Generated merging jdl \
1380 \n# $1 = full alien path to output directory to be merged \
1381 \n# $2 = train number \
1382 \n# $3 = production (like LHC10b) \
1383 \n# $4 = merging stage \
1384 \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1385 } else {
1386 sjdl1.Prepend("# Generated merging jdl \
1387 \n# $1 = full alien path to output directory to be merged \
1388 \n# $2 = merging stage \
1389 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1390 sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
1391 sjdl2.Prepend("# Generated merging jdl \
1392 \n# $1 = full alien path to output directory to be merged \
1393 \n# $2 = merging stage \
1394 \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
1395 }
1396 index = sjdl1.Index("JDLVariables");
1397 if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
1398 index = sjdl2.Index("JDLVariables");
1399 if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
1400 sjdl1 += "Workdirectorysize = {\"5000MB\"};";
1401 sjdl2 += "Workdirectorysize = {\"5000MB\"};";
1402 index = sjdl2.Index("Split =");
1403 if (index>=0) {
1404 index1 = sjdl2.Index("\n", index);
1405 sjdl2.Remove(index, index1-index+1);
1406 }
1407 index = sjdl2.Index("SplitMaxInputFileNumber");
1408 if (index>=0) {
1409 index1 = sjdl2.Index("\n", index);
1410 sjdl2.Remove(index, index1-index+1);
1411 }
1412 index = sjdl2.Index("InputDataCollection");
1413 if (index>=0) {
1414 index1 = sjdl2.Index(";", index);
1415 sjdl2.Remove(index, index1-index+1);
1416 }
1417 index = sjdl2.Index("InputDataListFormat");
1418 if (index>=0) {
1419 index1 = sjdl2.Index("\n", index);
1420 sjdl2.Remove(index, index1-index+1);
1421 }
1422 index = sjdl2.Index("InputDataList");
1423 if (index>=0) {
1424 index1 = sjdl2.Index("\n", index);
1425 sjdl2.Remove(index, index1-index+1);
1426 }
1427 sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
1428 // Write jdl to file
1429 ofstream out;
1430 out.open(fJDLName.Data(), ios::out);
1431 if (out.bad()) {
1432 Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
1433 return kFALSE;
1434 }
1435 out << sjdl << endl;
1436 out.close();
1437 TString mergeJDLName = fExecutable;
1438 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1439 if (fMergeViaJDL) {
1440 ofstream out1;
1441 out1.open(mergeJDLName.Data(), ios::out);
1442 if (out1.bad()) {
1443 Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
1444 return kFALSE;
1445 }
1446 out1 << sjdl1 << endl;
1447 out1.close();
1448 ofstream out2;
1449 TString finalJDL = mergeJDLName;
1450 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1451 out2.open(finalJDL.Data(), ios::out);
1452 if (out2.bad()) {
1453 Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
1454 return kFALSE;
1455 }
1456 out2 << sjdl2 << endl;
1457 out2.close();
1458 }
1459
1460 // Copy jdl to grid workspace
1461 if (!copy) {
1462 Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
1463 } else {
1464 TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
1465 TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1466 TString finalJDL = mergeJDLName;
1467 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1468 TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
1469 if (fProductionMode) {
1470 locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
1471 locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1472 locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
1473 }
1474 if (FileExists(locjdl)) gGrid->Rm(locjdl);
1475 if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1476 if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
1477 Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1478 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1479 if (fMergeViaJDL) {
1480 Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
1481 TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1482 TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
1483 }
1484 }
1485 return kTRUE;
1486}
1487
1488//______________________________________________________________________________
1489Bool_t AliAnalysisAlien::FileExists(const char *lfn)
1490{
1491// Returns true if file exists.
1492 if (!gGrid) return kFALSE;
1493 TString slfn = lfn;
1494 slfn.ReplaceAll("alien://","");
1495 TGridResult *res = gGrid->Ls(slfn);
1496 if (!res) return kFALSE;
1497 TMap *map = dynamic_cast<TMap*>(res->At(0));
1498 if (!map) {
1499 delete res;
1500 return kFALSE;
1501 }
1502 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
1503 if (!objs || !objs->GetString().Length()) {
1504 delete res;
1505 return kFALSE;
1506 }
1507 delete res;
1508 return kTRUE;
1509}
1510
1511//______________________________________________________________________________
1512Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
1513{
1514// Returns true if directory exists. Can be also a path.
1515 if (!gGrid) return kFALSE;
1516 // Check if dirname is a path
1517 TString dirstripped = dirname;
1518 dirstripped = dirstripped.Strip();
1519 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
1520 TString dir = gSystem->BaseName(dirstripped);
1521 dir += "/";
1522 TString path = gSystem->DirName(dirstripped);
1523 TGridResult *res = gGrid->Ls(path, "-F");
1524 if (!res) return kFALSE;
1525 TIter next(res);
1526 TMap *map;
1527 TObject *obj;
1528 while ((map=dynamic_cast<TMap*>(next()))) {
1529 obj = map->GetValue("name");
1530 if (!obj) break;
1531 if (dir == obj->GetName()) {
1532 delete res;
1533 return kTRUE;
1534 }
1535 }
1536 delete res;
1537 return kFALSE;
1538}
1539
1540//______________________________________________________________________________
1541void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
1542{
1543// Check input data type.
1544 isCollection = kFALSE;
1545 isXml = kFALSE;
1546 useTags = kFALSE;
1547 if (!gGrid) {
1548 Error("CheckDataType", "No connection to grid");
1549 return;
1550 }
1551 isCollection = IsCollection(lfn);
1552 TString msg = "\n##### file: ";
1553 msg += lfn;
1554 if (isCollection) {
1555 msg += " type: raw_collection;";
1556 // special treatment for collections
1557 isXml = kFALSE;
1558 // check for tag files in the collection
1559 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
1560 if (!res) {
1561 msg += " using_tags: No (unknown)";
1562 Info("CheckDataType", "%s", msg.Data());
1563 return;
1564 }
1565 const char* typeStr = res->GetKey(0, "origLFN");
1566 if (!typeStr || !strlen(typeStr)) {
1567 msg += " using_tags: No (unknown)";
1568 Info("CheckDataType", "%s", msg.Data());
1569 return;
1570 }
1571 TString file = typeStr;
1572 useTags = file.Contains(".tag");
1573 if (useTags) msg += " using_tags: Yes";
1574 else msg += " using_tags: No";
1575 Info("CheckDataType", "%s", msg.Data());
1576 return;
1577 }
1578 TString slfn(lfn);
1579 slfn.ToLower();
1580 isXml = slfn.Contains(".xml");
1581 if (isXml) {
1582 // Open xml collection and check if there are tag files inside
1583 msg += " type: xml_collection;";
1584 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
1585 if (!coll) {
1586 msg += " using_tags: No (unknown)";
1587 Info("CheckDataType", "%s", msg.Data());
1588 return;
1589 }
1590 TMap *map = coll->Next();
1591 if (!map) {
1592 msg += " using_tags: No (unknown)";
1593 Info("CheckDataType", "%s", msg.Data());
1594 return;
1595 }
1596 map = (TMap*)map->GetValue("");
1597 TString file;
1598 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
1599 useTags = file.Contains(".tag");
1600 delete coll;
1601 if (useTags) msg += " using_tags: Yes";
1602 else msg += " using_tags: No";
1603 Info("CheckDataType", "%s", msg.Data());
1604 return;
1605 }
1606 useTags = slfn.Contains(".tag");
1607 if (slfn.Contains(".root")) msg += " type: root file;";
1608 else msg += " type: unknown file;";
1609 if (useTags) msg += " using_tags: Yes";
1610 else msg += " using_tags: No";
1611 Info("CheckDataType", "%s", msg.Data());
1612}
1613
1614//______________________________________________________________________________
1615void AliAnalysisAlien::EnablePackage(const char *package)
1616{
1617// Enables a par file supposed to exist in the current directory.
1618 TString pkg(package);
1619 pkg.ReplaceAll(".par", "");
1620 pkg += ".par";
1621 if (gSystem->AccessPathName(pkg)) {
1622 Fatal("EnablePackage", "Package %s not found", pkg.Data());
1623 return;
1624 }
1625 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
1626 Info("EnablePackage", "AliEn plugin will use .par packages");
1627 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
1628 if (!fPackages) {
1629 fPackages = new TObjArray();
1630 fPackages->SetOwner();
1631 }
1632 fPackages->Add(new TObjString(pkg));
1633}
1634
1635//______________________________________________________________________________
1636TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
1637{
1638// Make a tree from files having the location specified in fFileForTestMode.
1639// Inspired from JF's CreateESDChain.
1640 if (fFileForTestMode.IsNull()) {
1641 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
1642 return NULL;
1643 }
1644 if (gSystem->AccessPathName(fFileForTestMode)) {
1645 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
1646 return NULL;
1647 }
1648 // Open the file
1649 ifstream in;
1650 in.open(fFileForTestMode);
1651 Int_t count = 0;
1652 // Read the input list of files and add them to the chain
1653 TString line;
1654 TChain *chain = new TChain(treeName);
1655 while (in.good())
1656 {
1657 in >> line;
1658 if (line.IsNull()) continue;
1659 if (count++ == fNtestFiles) break;
1660 TString esdFile(line);
1661 TFile *file = TFile::Open(esdFile);
1662 if (file) {
1663 if (!file->IsZombie()) chain->Add(esdFile);
1664 file->Close();
1665 } else {
1666 Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
1667 }
1668 }
1669 in.close();
1670 if (!chain->GetListOfFiles()->GetEntries()) {
1671 Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
1672 delete chain;
1673 return NULL;
1674 }
1675// chain->ls();
1676 return chain;
1677}
1678
1679//______________________________________________________________________________
1680const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
1681{
1682// Get job status for all jobs with jobid>jobidstart.
1683 static char mstatus[20];
1684 mstatus[0] = '\0';
1685 nrunning = 0;
1686 nwaiting = 0;
1687 nerror = 0;
1688 ndone = 0;
1689 TGridJobStatusList *list = gGrid->Ps("");
1690 if (!list) return mstatus;
1691 Int_t nentries = list->GetSize();
1692 TGridJobStatus *status;
1693 Int_t pid;
1694 for (Int_t ijob=0; ijob<nentries; ijob++) {
1695 status = (TGridJobStatus *)list->At(ijob);
1696 pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
1697 if (pid<jobidstart) continue;
1698 if (pid == lastid) {
1699 gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
1700 }
1701 switch (status->GetStatus()) {
1702 case TGridJobStatus::kWAITING:
1703 nwaiting++; break;
1704 case TGridJobStatus::kRUNNING:
1705 nrunning++; break;
1706 case TGridJobStatus::kABORTED:
1707 case TGridJobStatus::kFAIL:
1708 case TGridJobStatus::kUNKNOWN:
1709 nerror++; break;
1710 case TGridJobStatus::kDONE:
1711 ndone++;
1712 }
1713 }
1714 list->Delete();
1715 delete list;
1716 return mstatus;
1717}
1718
1719//______________________________________________________________________________
1720Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
1721{
1722// Returns true if file is a collection. Functionality duplicated from
1723// TAlien::Type() because we don't want to directly depend on TAlien.
1724 if (!gGrid) {
1725 Error("IsCollection", "No connection to grid");
1726 return kFALSE;
1727 }
1728 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
1729 if (!res) return kFALSE;
1730 const char* typeStr = res->GetKey(0, "type");
1731 if (!typeStr || !strlen(typeStr)) return kFALSE;
1732 if (!strcmp(typeStr, "collection")) return kTRUE;
1733 delete res;
1734 return kFALSE;
1735}
1736
1737//______________________________________________________________________________
1738Bool_t AliAnalysisAlien::IsSingleOutput() const
1739{
1740// Check if single-ouput option is on.
1741 return (!fOutputSingle.IsNull());
1742}
1743
1744//______________________________________________________________________________
1745void AliAnalysisAlien::Print(Option_t *) const
1746{
1747// Print current plugin settings.
1748 printf("### AliEn analysis plugin current settings ###\n");
1749 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1750 if (mgr && mgr->IsProofMode()) {
1751 TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
1752 if (TestBit(AliAnalysisGrid::kTest))
1753 proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
1754 printf("%s %s\n", proofType.Data(), fProofCluster.Data());
1755 if (!fProofDataSet.IsNull())
1756 printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
1757 if (fProofReset==1)
1758 printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1759 if (fProofReset>1)
1760 printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
1761 if (!fRootVersionForProof.IsNull())
1762 printf("= ROOT version requested________________________ %s\n", fRootVersionForProof.Data());
1763 else
1764 printf("= ROOT version requested________________________ default\n");
1765 printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
1766 if (!fAliRootMode.IsNull())
1767 printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
1768 if (fNproofWorkers)
1769 printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
1770 if (fNproofWorkersPerSlave)
1771 printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
1772 if (TestSpecialBit(kClearPackages))
1773 printf("= ClearPackages requested...\n");
1774 if (fIncludePath.Data())
1775 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1776 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1777 if (fPackages && fPackages->GetEntries()) {
1778 TIter next(fPackages);
1779 TObject *obj;
1780 TString list;
1781 while ((obj=next())) list += obj->GetName();
1782 printf("= Par files to be used: ________________________ %s\n", list.Data());
1783 }
1784 if (TestSpecialBit(kProofConnectGrid))
1785 printf("= Requested PROOF connection to grid\n");
1786 return;
1787 }
1788 printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
1789 if (fOverwriteMode) {
1790 printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
1791 \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
1792 }
1793 printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
1794 printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO");
1795 printf("= Production mode:______________________________ %d\n", fProductionMode);
1796 printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
1797 printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
1798 printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
1799 if (fUser.Length())
1800 printf("= User running the plugin: _____________________ %s\n", fUser.Data());
1801 printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
1802 printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
1803 printf("= Data base directory path requested: __________ %s\n", fGridDataDir.Data());
1804 printf("= Data search pattern: _________________________ %s\n", fDataPattern.Data());
1805 printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
1806 if (fRunNumbers.Length())
1807 printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
1808 if (fRunRange[0])
1809 printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
1810 if (!fRunRange[0] && !fRunNumbers.Length()) {
1811 TIter next(fInputFiles);
1812 TObject *obj;
1813 TString list;
1814 while ((obj=next())) list += obj->GetName();
1815 printf("= Input files to be processed: _________________ %s\n", list.Data());
1816 }
1817 if (TestBit(AliAnalysisGrid::kTest))
1818 printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
1819 printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
1820 printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
1821 printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
1822 printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
1823 printf("=====================================================================\n");
1824 printf("= Job price: ___________________________________ %d\n", fPrice);
1825 printf("= Time to live (TTL): __________________________ %d\n", fTTL);
1826 printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
1827 if (fMaxInitFailed>0)
1828 printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
1829 if (fMasterResubmitThreshold>0)
1830 printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
1831 printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
1832 if (fNrunsPerMaster>0)
1833 printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
1834 printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
1835 printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
1836 printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
1837 if (fArguments.Length())
1838 printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
1839 if (fExecutableArgs.Length())
1840 printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
1841 printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
1842 printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
1843 printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
1844 printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
1845 if (fDatasetName)
1846 printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
1847 printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
1848 if (fIncludePath.Data())
1849 printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
1850 if (fCloseSE.Length())
1851 printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
1852 if (fFriendChainName.Length())
1853 printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
1854 if (fPackages && fPackages->GetEntries()) {
1855 TIter next(fPackages);
1856 TObject *obj;
1857 TString list;
1858 while ((obj=next())) list += obj->GetName();
1859 printf("= Par files to be used: ________________________ %s\n", list.Data());
1860 }
1861}
1862
1863//______________________________________________________________________________
1864void AliAnalysisAlien::SetDefaults()
1865{
1866// Set default values for everything. What cannot be filled will be left empty.
1867 if (fGridJDL) delete fGridJDL;
1868 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1869 fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
1870 fPrice = 1;
1871 fTTL = 30000;
1872 fSplitMaxInputFileNumber = 100;
1873 fMaxInitFailed = 0;
1874 fMasterResubmitThreshold = 0;
1875 fNtestFiles = 10;
1876 fNreplicas = 2;
1877 fRunRange[0] = 0;
1878 fRunRange[1] = 0;
1879 fRunPrefix = "%d";
1880 fNrunsPerMaster = 1;
1881 fMaxMergeFiles = 100;
1882 fRunNumbers = "";
1883 fExecutable = "analysis.sh";
1884 fExecutableCommand = "root -b -q";
1885 fArguments = "";
1886 fExecutableArgs = "";
1887 fAnalysisMacro = "myAnalysis.C";
1888 fAnalysisSource = "";
1889 fAdditionalLibs = "";
1890 fSplitMode = "se";
1891 fAPIVersion = "";
1892 fROOTVersion = "";
1893 fAliROOTVersion = "";
1894 fUser = ""; // Your alien user name
1895 fGridWorkingDir = "";
1896 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
1897 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
1898 fFriendChainName = "";
1899 fGridOutputDir = "output";
1900 fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
1901 fOutputFiles = ""; // Like "AliAODs.root histos.root"
1902 fInputFormat = "xml-single";
1903 fJDLName = "analysis.jdl";
1904 fJobTag = "Automatically generated analysis JDL";
1905 fMergeExcludes = "";
1906 fMergeViaJDL = 0;
1907 SetUseCopy(kTRUE);
1908 SetCheckCopy(kTRUE);
1909 SetDefaultOutputs(kTRUE);
1910 fOverwriteMode = 1;
1911}
1912
1913//______________________________________________________________________________
1914Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
1915{
1916// Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
1917 // First check if the result is already in the output directory.
1918 if (FileExists(Form("%s/%s",aliendir,filename))) {
1919 printf("Final merged results found. Not merging again.\n");
1920 return kFALSE;
1921 }
1922 // Now check the last stage done.
1923 Int_t stage = 0;
1924 while (1) {
1925 if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
1926 stage++;
1927 }
1928 // Next stage of merging
1929 stage++;
1930 TString pattern = "*root_archive.zip";
1931 if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
1932 TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
1933 if (res) delete res;
1934 // Write standard output to file
1935 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
1936 // Count the number of files inside
1937 ifstream ifile;
1938 ifile.open(Form("Stage_%d.xml",stage));
1939 if (!ifile.good()) {
1940 ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
1941 return kFALSE;
1942 }
1943 TString line;
1944 Int_t nfiles = 0;
1945 while (!ifile.eof()) {
1946 ifile >> line;
1947 if (line.Contains("/event")) nfiles++;
1948 }
1949 ifile.close();
1950 if (!nfiles) {
1951 ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
1952 return kFALSE;
1953 } else {
1954 printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
1955 }
1956 // Copy the file in the output directory
1957 printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
1958 TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
1959 // Check if this is the last stage to be done.
1960 Bool_t laststage = (nfiles<nperchunk);
1961 if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
1962 if (laststage) {
1963 printf("### Submiting final merging stage %d\n", stage);
1964 TString finalJDL = jdl;
1965 finalJDL.ReplaceAll(".jdl", "_final.jdl");
1966 TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
1967 Int_t jobId = SubmitSingleJob(query);
1968 if (!jobId) return kFALSE;
1969 } else {
1970 printf("### Submiting merging stage %d\n", stage);
1971 TString query = Form("submit %s %s %d", jdl, aliendir, stage);
1972 Int_t jobId = SubmitSingleJob(query);
1973 if (!jobId) return kFALSE;
1974 }
1975 return kTRUE;
1976}
1977
1978//______________________________________________________________________________
1979Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
1980{
1981// Submits a single job corresponding to the query and returns job id. If 0 submission failed.
1982 if (!gGrid) return 0;
1983 printf("=> %s ------> ",query);
1984 TGridResult *res = gGrid->Command(query);
1985 if (!res) return 0;
1986 TString jobId = res->GetKey(0,"jobId");
1987 delete res;
1988 if (jobId.IsNull()) {
1989 printf("submission failed. Reason:\n");
1990 gGrid->Stdout();
1991 gGrid->Stderr();
1992 ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
1993 return 0;
1994 }
1995 printf(" Job id: %s\n", jobId.Data());
1996 return atoi(jobId);
1997}
1998
1999//______________________________________________________________________________
2000Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
2001{
2002// Merge given output files from basedir. Basedir can be an alien output directory
2003// but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2004// files in a group (ignored for xml input). Merging can be done in stages:
2005// stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2006// stage=1 : works with an xml of all root_archive.zip in the output directory
2007// stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
2008 TString outputFile = output;
2009 TString command;
2010 TString outputChunk;
2011 TString previousChunk = "";
2012 TObjArray *listoffiles = new TObjArray();
2013// listoffiles->SetOwner();
2014 Int_t countChunk = 0;
2015 Int_t countZero = nmaxmerge;
2016 Bool_t merged = kTRUE;
2017 Int_t index = outputFile.Index("@");
2018 if (index > 0) outputFile.Remove(index);
2019 TString inputFile = outputFile;
2020 TString sbasedir = basedir;
2021 if (sbasedir.Contains(".xml")) {
2022 // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2023 nmaxmerge = 9999999;
2024 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2025 if (!coll) {
2026 ::Error("MergeOutput", "Input XML collection empty.");
2027 return kFALSE;
2028 }
2029 // Iterate grid collection
2030 while (coll->Next()) {
2031 TString fname = gSystem->DirName(coll->GetTURL());
2032 fname += "/";
2033 fname += inputFile;
2034 listoffiles->Add(new TNamed(fname.Data(),""));
2035 }
2036 } else {
2037 command = Form("find %s/ *%s", basedir, inputFile.Data());
2038 printf("command: %s\n", command.Data());
2039 TGridResult *res = gGrid->Command(command);
2040 if (!res) {
2041 ::Error("MergeOutput","No result for the find command\n");
2042 delete listoffiles;
2043 return kFALSE;
2044 }
2045 TIter nextmap(res);
2046 TMap *map = 0;
2047 while ((map=(TMap*)nextmap())) {
2048 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2049 if (!objs || !objs->GetString().Length()) {
2050 // Nothing found - skip this output
2051 delete res;
2052 delete listoffiles;
2053 return kFALSE;
2054 }
2055 listoffiles->Add(new TNamed(objs->GetName(),""));
2056 }
2057 delete res;
2058 }
2059 if (!listoffiles->GetEntries()) {
2060 ::Error("MergeOutput","No result for the find command\n");
2061 delete listoffiles;
2062 return kFALSE;
2063 }
2064
2065 TFileMerger *fm = 0;
2066 TIter next0(listoffiles);
2067 TObjArray *listoffilestmp = new TObjArray();
2068 listoffilestmp->SetOwner();
2069 TObject *nextfile;
2070 TString snextfile;
2071 // Keep only the files at upper level
2072 Int_t countChar = 0;
2073 while ((nextfile=next0())) {
2074 snextfile = nextfile->GetName();
2075 Int_t crtCount = snextfile.CountChar('/');
2076 if (nextfile == listoffiles->First()) countChar = crtCount;
2077 if (crtCount < countChar) countChar = crtCount;
2078 }
2079 next0.Reset();
2080 while ((nextfile=next0())) {
2081 snextfile = nextfile->GetName();
2082 Int_t crtCount = snextfile.CountChar('/');
2083 if (crtCount > countChar) {
2084 delete nextfile;
2085 continue;
2086 }
2087 listoffilestmp->Add(nextfile);
2088 }
2089 delete listoffiles;
2090 listoffiles = listoffilestmp; // Now contains 'good' files
2091 listoffiles->Print();
2092 TIter next(listoffiles);
2093 // Check if there is a merge operation to resume. Works only for stage 0 or 1.
2094 outputChunk = outputFile;
2095 outputChunk.ReplaceAll(".root", "_*.root");
2096 // Check for existent temporary merge files
2097 // Check overwrite mode and remove previous partial results if needed
2098 // Preserve old merging functionality for stage 0.
2099 if (stage==0) {
2100 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2101 while (1) {
2102 // Skip as many input files as in a chunk
2103 for (Int_t counter=0; counter<nmaxmerge; counter++) {
2104 nextfile = next();
2105 if (!nextfile) {
2106 ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
2107 delete listoffiles;
2108 return kFALSE;
2109 }
2110 snextfile = nextfile->GetName();
2111 }
2112 outputChunk = outputFile;
2113 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2114 countChunk++;
2115 if (gSystem->AccessPathName(outputChunk)) continue;
2116 // Merged file with chunks up to <countChunk> found
2117 ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
2118 previousChunk = outputChunk;
2119 break;
2120 }
2121 }
2122 countZero = nmaxmerge;
2123
2124 while ((nextfile=next())) {
2125 snextfile = nextfile->GetName();
2126 // Loop 'find' results and get next LFN
2127 if (countZero == nmaxmerge) {
2128 // First file in chunk - create file merger and add previous chunk if any.
2129 fm = new TFileMerger(kFALSE);
2130 fm->SetFastMethod(kTRUE);
2131 if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
2132 outputChunk = outputFile;
2133 outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
2134 }
2135 // If last file found, put merged results in the output file
2136 if (nextfile == listoffiles->Last()) outputChunk = outputFile;
2137 // Add file to be merged and decrement chunk counter.
2138 fm->AddFile(snextfile);
2139 countZero--;
2140 if (countZero==0 || nextfile == listoffiles->Last()) {
2141 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2142 // Nothing found - skip this output
2143 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2144 merged = kFALSE;
2145 break;
2146 }
2147 fm->OutputFile(outputChunk);
2148 // Merge the outputs, then go to next chunk
2149 if (!fm->Merge()) {
2150 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2151 merged = kFALSE;
2152 break;
2153 } else {
2154 ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
2155 gSystem->Unlink(previousChunk);
2156 }
2157 if (nextfile == listoffiles->Last()) break;
2158 countChunk++;
2159 countZero = nmaxmerge;
2160 previousChunk = outputChunk;
2161 }
2162 }
2163 delete listoffiles;
2164 delete fm;
2165 return merged;
2166 }
2167 // Merging stage different than 0.
2168 // Move to the begining of the requested chunk.
2169 fm = new TFileMerger(kFALSE);
2170 fm->SetFastMethod(kTRUE);
2171 while ((nextfile=next())) fm->AddFile(nextfile->GetName());
2172 delete listoffiles;
2173 if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
2174 // Nothing found - skip this output
2175 ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
2176 delete fm;
2177 return kFALSE;
2178 }
2179 fm->OutputFile(outputFile);
2180 // Merge the outputs
2181 if (!fm->Merge()) {
2182 ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
2183 delete fm;
2184 return kFALSE;
2185 } else {
2186 ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
2187 }
2188 delete fm;
2189 return kTRUE;
2190}
2191
2192//______________________________________________________________________________
2193Bool_t AliAnalysisAlien::MergeOutputs()
2194{
2195// Merge analysis outputs existing in the AliEn space.
2196 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
2197 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2198 if (!Connect()) {
2199 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
2200 return kFALSE;
2201 }
2202 if (fMergeViaJDL) {
2203 if (!TestBit(AliAnalysisGrid::kMerge)) {
2204 Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
2205 return kFALSE;
2206 }
2207 if (fProductionMode) {
2208 Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
2209 return kFALSE;
2210 }
2211 Info("MergeOutputs", "Submitting merging JDL");
2212 if (!SubmitMerging()) return kFALSE;
2213 Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
2214 Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
2215 return kFALSE;
2216 }
2217 // Get the output path
2218 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2219 if (!DirectoryExists(fGridOutputDir)) {
2220 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
2221 return kFALSE;
2222 }
2223 if (!fOutputFiles.Length()) {
2224 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
2225 return kFALSE;
2226 }
2227 // Check if fast read option was requested
2228 Info("MergeOutputs", "Started local merging of output files from: alien://%s \
2229 \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
2230 if (fFastReadOption) {
2231 Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
2232 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
2233 gEnv->SetValue("XNet.ConnectTimeout",50);
2234 gEnv->SetValue("XNet.RequestTimeout",50);
2235 gEnv->SetValue("XNet.MaxRedirectCount",2);
2236 gEnv->SetValue("XNet.ReconnectTimeout",50);
2237 gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
2238 }
2239 // Make sure we change the temporary directory
2240 gSystem->Setenv("TMPDIR", gSystem->pwd());
2241 TObjArray *list = fOutputFiles.Tokenize(",");
2242 TIter next(list);
2243 TObjString *str;
2244 TString outputFile;
2245 Bool_t merged = kTRUE;
2246 while((str=(TObjString*)next())) {
2247 outputFile = str->GetString();
2248 Int_t index = outputFile.Index("@");
2249 if (index > 0) outputFile.Remove(index);
2250 TString outputChunk = outputFile;
2251 outputChunk.ReplaceAll(".root", "_*.root");
2252 // Skip already merged outputs
2253 if (!gSystem->AccessPathName(outputFile)) {
2254 if (fOverwriteMode) {
2255 Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
2256 gSystem->Unlink(outputFile);
2257 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2258 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
2259 outputChunk.Data());
2260 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
2261 }
2262 } else {
2263 Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
2264 continue;
2265 }
2266 } else {
2267 if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
2268 Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
2269 outputChunk.Data());
2270 gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
2271 }
2272 }
2273 if (fMergeExcludes.Length() &&
2274 fMergeExcludes.Contains(outputFile.Data())) continue;
2275 // Perform a 'find' command in the output directory, looking for registered outputs
2276 merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
2277 if (!merged) {
2278 Error("MergeOutputs", "Terminate() will NOT be executed");
2279 return kFALSE;
2280 }
2281 TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
2282 if (fileOpened) fileOpened->Close();
2283 }
2284 return kTRUE;
2285}
2286
2287//______________________________________________________________________________
2288void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
2289{
2290// Use the output files connected to output containers from the analysis manager
2291// rather than the files defined by SetOutputFiles
2292 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
2293 Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
2294 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
2295}
2296
2297//______________________________________________________________________________
2298void AliAnalysisAlien::SetOutputFiles(const char *list)
2299{
2300// Manually set the output files list.
2301// Removes duplicates. Not allowed if default outputs are not disabled.
2302 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2303 Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
2304 return;
2305 }
2306 Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
2307 fOutputFiles = "";
2308 TString slist = list;
2309 if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
2310 TObjArray *arr = slist.Tokenize(" ");
2311 TObjString *os;
2312 TIter next(arr);
2313 TString sout;
2314 while ((os=(TObjString*)next())) {
2315 sout = os->GetString();
2316 if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
2317 if (fOutputFiles.Contains(sout)) continue;
2318 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2319 fOutputFiles += sout;
2320 }
2321 delete arr;
2322}
2323
2324//______________________________________________________________________________
2325void AliAnalysisAlien::SetOutputArchive(const char *list)
2326{
2327// Manually set the output archive list. Free text - you are on your own...
2328// Not allowed if default outputs are not disabled.
2329 if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2330 Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
2331 return;
2332 }
2333 Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
2334 fOutputArchive = list;
2335}
2336
2337//______________________________________________________________________________
2338void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
2339{
2340// Setting a prefered output SE is not allowed anymore.
2341 Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
2342}
2343
2344//______________________________________________________________________________
2345Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
2346{
2347// Start remote grid analysis.
2348 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2349 Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
2350 if (!mgr || !mgr->IsInitialized()) {
2351 Error("StartAnalysis", "You need an initialized analysis manager for this");
2352 return kFALSE;
2353 }
2354 // Are we in PROOF mode ?
2355 if (mgr->IsProofMode()) {
2356 Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
2357 if (fProofCluster.IsNull()) {
2358 Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
2359 return kFALSE;
2360 }
2361 if (fProofDataSet.IsNull() && !testMode) {
2362 Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
2363 return kFALSE;
2364 }
2365 // Set the needed environment
2366 gEnv->SetValue("XSec.GSI.DelegProxy","2");
2367 // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
2368 if (fProofReset && !testMode) {
2369 if (fProofReset==1) {
2370 Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
2371 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
2372 } else {
2373 Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
2374 gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
2375 }
2376 Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
2377 return kFALSE;
2378 }
2379 // Do we need to change the ROOT version ? The success of this cannot be checked.
2380 if (!fRootVersionForProof.IsNull() && !testMode) {
2381 gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"%s\");",
2382 fProofCluster.Data(), fRootVersionForProof.Data()));
2383 }
2384 // Connect to PROOF and check the status
2385 Long_t proof = 0;
2386 TString sworkers;
2387 if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
2388 else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
2389 if (!testMode) {
2390 if (!sworkers.IsNull())
2391 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
2392 else
2393 proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
2394 } else {
2395 proof = gROOT->ProcessLine("TProof::Open(\"\");");
2396 if (!proof) {
2397 Error("StartAnalysis", "Could not start PROOF in test mode");
2398 return kFALSE;
2399 }
2400 }
2401 if (!proof) {
2402 Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
2403 return kFALSE;
2404 }
2405 if (fNproofWorkersPerSlave*fNproofWorkers > 0)
2406 gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
2407 // Is dataset existing ?
2408 if (!testMode) {
2409 TString dataset = fProofDataSet;
2410 Int_t index = dataset.Index("#");
2411 if (index>=0) dataset.Remove(index);
2412// if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
2413// Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
2414// return kFALSE;
2415// }
2416// Info("StartAnalysis", "Dataset %s found", dataset.Data());
2417 }
2418 // Is ClearPackages() needed ?
2419 if (TestSpecialBit(kClearPackages)) {
2420 Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
2421 gROOT->ProcessLine("gProof->ClearPackages();");
2422 }
2423 // Is a given aliroot mode requested ?
2424 TList optionsList;
2425 TString parLibs;
2426 if (!fAliRootMode.IsNull()) {
2427 TString alirootMode = fAliRootMode;
2428 if (alirootMode == "default") alirootMode = "";
2429 Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
2430 optionsList.SetOwner();
2431 optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
2432 // Check the additional libs to be loaded
2433 TString extraLibs;
2434 Bool_t parMode = kFALSE;
2435 if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice";
2436 // Parse the extra libs for .so
2437 if (fAdditionalLibs.Length()) {
2438 TObjArray *list = fAdditionalLibs.Tokenize(" ");
2439 TIter next(list);
2440 TObjString *str;
2441 while((str=(TObjString*)next())) {
2442 if (str->GetString().Contains(".so")) {
2443 if (parMode) {
2444 Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
2445 break;
2446 }
2447 TString stmp = str->GetName();
2448 if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
2449 stmp.ReplaceAll(".so","");
2450 if (!extraLibs.IsNull()) extraLibs += ":";
2451 extraLibs += stmp;
2452 continue;
2453 }
2454 if (str->GetString().Contains(".par")) {
2455 // The first par file found in the list will not allow any further .so
2456 parMode = kTRUE;
2457 if (!parLibs.IsNull()) parLibs += ":";
2458 parLibs += str->GetName();
2459 continue;
2460 }
2461 }
2462 if (list) delete list;
2463 }
2464 if (!extraLibs.IsNull()) optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
2465 // Check extra includes
2466 if (!fIncludePath.IsNull()) {
2467 TString includePath = fIncludePath;
2468 includePath.ReplaceAll(" ",":");
2469 includePath.ReplaceAll("$ALICE_ROOT","");
2470 includePath.ReplaceAll("${ALICE_ROOT}","");
2471 includePath.ReplaceAll("-I","");
2472 includePath.Strip(TString::kTrailing, ':');
2473 Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
2474 optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
2475 }
2476 // Check if connection to grid is requested
2477 if (TestSpecialBit(kProofConnectGrid))
2478 optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
2479 // Enable AliRoot par
2480 if (testMode) {
2481 // Enable proof lite package
2482 TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
2483 for (Int_t i=0; i<optionsList.GetSize(); i++) {
2484 TNamed *obj = (TNamed*)optionsList.At(i);
2485 printf("%s %s\n", obj->GetName(), obj->GetTitle());
2486 }
2487 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
2488 && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
2489 Info("StartAnalysis", "AliRootProofLite enabled");
2490 } else {
2491 Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
2492 return kFALSE;
2493 }
2494 } else {
2495 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
2496 fAliROOTVersion.Data(), &optionsList))) {
2497 Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
2498 return kFALSE;
2499 }
2500 }
2501 // Enable first par files from fAdditionalLibs
2502 if (!parLibs.IsNull()) {
2503 TObjArray *list = parLibs.Tokenize(":");
2504 TIter next(list);
2505 TObjString *package;
2506 while((package=(TObjString*)next())) {
2507 TString spkg = package->GetName();
2508 spkg.ReplaceAll(".par", "");
2509 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
2510 if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
2511 TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
2512 if (gROOT->ProcessLine(enablePackage)) {
2513 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2514 return kFALSE;
2515 }
2516 } else {
2517 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2518 return kFALSE;
2519 }
2520 }
2521 if (list) delete list;
2522 }
2523 } else {
2524 if (fAdditionalLibs.Contains(".so") && !testMode) {
2525 Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
2526 \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
2527 return kFALSE;
2528 }
2529 }
2530 // Enable par files if requested
2531 if (fPackages && fPackages->GetEntries()) {
2532 TIter next(fPackages);
2533 TObject *package;
2534 while ((package=next())) {
2535 // Skip packages already enabled
2536 if (parLibs.Contains(package->GetName())) continue;
2537 TString spkg = package->GetName();
2538 spkg.ReplaceAll(".par", "");
2539 gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
2540 if (gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
2541 if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
2542 Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
2543 return kFALSE;
2544 }
2545 } else {
2546 Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
2547 return kFALSE;
2548 }
2549 }
2550 }
2551 // Do we need to load analysis source files ?
2552 // NOTE: don't load on client since this is anyway done by the user to attach his task.
2553 if (fAnalysisSource.Length()) {
2554 TObjArray *list = fAnalysisSource.Tokenize(" ");
2555 TIter next(list);
2556 TObjString *str;
2557 while((str=(TObjString*)next())) {
2558 gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
2559 }
2560 if (list) delete list;
2561 }
2562 if (testMode) {
2563 // Register dataset to proof lite.
2564 if (fFileForTestMode.IsNull()) {
2565 Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2566 return kFALSE;
2567 }
2568 if (gSystem->AccessPathName(fFileForTestMode)) {
2569 Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2570 return kFALSE;
2571 }
2572 TFileCollection *coll = new TFileCollection();
2573 coll->AddFromFile(fFileForTestMode);
2574 gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
2575 gROOT->ProcessLine("gProof->ShowDataSets()");
2576 }
2577 return kTRUE;
2578 }
2579
2580 // Check if output files have to be taken from the analysis manager
2581 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
2582 // Add output files and AOD files
2583 fOutputFiles = GetListOfFiles("outaod");
2584 // Add extra files registered to the analysis manager
2585 TString extra = GetListOfFiles("ext");
2586 if (!extra.IsNull()) {
2587 extra.ReplaceAll(".root", "*.root");
2588 if (!fOutputFiles.IsNull()) fOutputFiles += ",";
2589 fOutputFiles += extra;
2590 }
2591 // Compose the output archive.
2592 fOutputArchive = "log_archive.zip:std*@disk=1 ";
2593 fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
2594 }
2595// if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
2596 if (TestBit(AliAnalysisGrid::kOffline)) {
2597 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
2598 \n there nor any job run. You can revise the JDL and analysis \
2599 \n macro then run the same in \"submit\" mode.");
2600 } else if (TestBit(AliAnalysisGrid::kTest)) {
2601 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
2602 \n dataset.");
2603 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
2604 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
2605 \n space and job submitted.");
2606 } else if (TestBit(AliAnalysisGrid::kMerge)) {
2607 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
2608 if (fMergeViaJDL) CheckInputData();
2609 return kTRUE;
2610 } else {
2611 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
2612 }
2613
2614 Print();
2615 if (!Connect()) {
2616 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
2617 return kFALSE;
2618 }
2619 if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
2620 if (!CheckInputData()) {
2621 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
2622 return kFALSE;
2623 }
2624 if (!CreateDataset(fDataPattern)) {
2625 TString serror;
2626 if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
2627 if (fRunNumbers.Length()) serror = "run numbers";
2628 if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
2629 serror += Form("\n or data pattern <%s>", fDataPattern.Data());
2630 Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
2631 return kFALSE;
2632 }
2633 WriteAnalysisFile();
2634 WriteAnalysisMacro();
2635 WriteExecutable();
2636 WriteValidationScript();
2637 if (fMergeViaJDL) {
2638 WriteMergingMacro();
2639 WriteMergeExecutable();
2640 WriteValidationScript(kTRUE);
2641 }
2642 if (!CreateJDL()) return kFALSE;
2643 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
2644 if (testMode) {
2645 // Locally testing the analysis
2646 Info("StartAnalysis", "\n_______________________________________________________________________ \
2647 \n Running analysis script in a daughter shell as on a worker node \
2648 \n_______________________________________________________________________");
2649 TObjArray *list = fOutputFiles.Tokenize(",");
2650 TIter next(list);
2651 TObjString *str;
2652 TString outputFile;
2653 while((str=(TObjString*)next())) {
2654 outputFile = str->GetString();
2655 Int_t index = outputFile.Index("@");
2656 if (index > 0) outputFile.Remove(index);
2657 if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
2658 }
2659 delete list;
2660 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
2661 gSystem->Exec(Form("bash %s",fValidationScript.Data()));
2662// gSystem->Exec("cat stdout");
2663 return kFALSE;
2664 }
2665 // Check if submitting is managed by LPM manager
2666 if (fProductionMode) {
2667 TString prodfile = fJDLName;
2668 prodfile.ReplaceAll(".jdl", ".prod");
2669 WriteProductionFile(prodfile);
2670 Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
2671 return kFALSE;
2672 }
2673 // Submit AliEn job(s)
2674 gGrid->Cd(fGridOutputDir);
2675 TGridResult *res;
2676 TString jobID = "";
2677 if (!fRunNumbers.Length() && !fRunRange[0]) {
2678 // Submit a given xml or a set of runs
2679 res = gGrid->Command(Form("submit %s", fJDLName.Data()));
2680 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
2681 if (res) {
2682 const char *cjobId = res->GetKey(0,"jobId");
2683 if (!cjobId) {
2684 gGrid->Stdout();
2685 gGrid->Stderr();
2686 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
2687 return kFALSE;
2688 } else {
2689 Info("StartAnalysis", "\n_______________________________________________________________________ \
2690 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
2691 \n_______________________________________________________________________",
2692 fJDLName.Data(), cjobId);
2693 jobID = cjobId;
2694 }
2695 delete res;
2696 } else {
2697 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2698 return kFALSE;
2699 }
2700 } else {
2701 // Submit for a range of enumeration of runs.
2702 if (!Submit()) return kFALSE;
2703 }
2704
2705 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
2706 \n You may exit at any time and terminate the job later using the option <terminate> \
2707 \n ##################################################################################", jobID.Data());
2708 gSystem->Exec("aliensh");
2709 return kTRUE;
2710}
2711
2712//______________________________________________________________________________
2713const char *AliAnalysisAlien::GetListOfFiles(const char *type)
2714{
2715// Get a comma-separated list of output files of the requested type.
2716// Type can be (case unsensitive):
2717// aod - list of aod files (std, extensions and filters)
2718// out - list of output files connected to containers (but not aod's or extras)
2719// ext - list of extra files registered to the manager
2720// ter - list of files produced in terminate
2721 static TString files;
2722 files = "";
2723 TString stype = type;
2724 stype.ToLower();
2725 TString aodfiles, extra;
2726 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2727 if (!mgr) {
2728 ::Error("GetListOfFiles", "Cannot call this without analysis manager");
2729 return files.Data();
2730 }
2731 if (mgr->GetOutputEventHandler()) {
2732 aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
2733 TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs();
2734 if (!extraaod.IsNull()) {
2735 aodfiles += ",";
2736 aodfiles += extraaod;
2737 }
2738 }
2739 if (stype.Contains("aod")) {
2740 files = aodfiles;
2741 if (stype == "aod") return files.Data();
2742 }
2743 // Add output files that are not in the list of AOD files
2744 TString outputfiles = "";
2745 TIter next(mgr->GetOutputs());
2746 AliAnalysisDataContainer *output;
2747 const char *filename = 0;
2748 while ((output=(AliAnalysisDataContainer*)next())) {
2749 filename = output->GetFileName();
2750 if (!(strcmp(filename, "default"))) continue;
2751 if (outputfiles.Contains(filename)) continue;
2752 if (aodfiles.Contains(filename)) continue;
2753 if (!outputfiles.IsNull()) outputfiles += ",";
2754 outputfiles += filename;
2755 }
2756 if (stype.Contains("out")) {
2757 if (!files.IsNull()) files += ",";
2758 files += outputfiles;
2759 if (stype == "out") return files.Data();
2760 }
2761 // Add extra files registered to the analysis manager
2762 TString sextra;
2763 extra = mgr->GetExtraFiles();
2764 if (!extra.IsNull()) {
2765 extra.Strip();
2766 extra.ReplaceAll(" ", ",");
2767 TObjArray *fextra = extra.Tokenize(",");
2768 TIter nextx(fextra);
2769 TObject *obj;
2770 while ((obj=nextx())) {
2771 if (aodfiles.Contains(obj->GetName())) continue;
2772 if (outputfiles.Contains(obj->GetName())) continue;
2773 if (sextra.Contains(obj->GetName())) continue;
2774 if (!sextra.IsNull()) sextra += ",";
2775 sextra += obj->GetName();
2776 }
2777 delete fextra;
2778 if (stype.Contains("ext")) {
2779 if (!files.IsNull()) files += ",";
2780 files += sextra;
2781 }
2782 }
2783 if (stype == "ext") return files.Data();
2784 TString termfiles;
2785 if (!fTerminateFiles.IsNull()) {
2786 fTerminateFiles.Strip();
2787 fTerminateFiles.ReplaceAll(" ",",");
2788 TObjArray *fextra = fTerminateFiles.Tokenize(",");
2789 TIter nextx(fextra);
2790 TObject *obj;
2791 while ((obj=nextx())) {
2792 if (aodfiles.Contains(obj->GetName())) continue;
2793 if (outputfiles.Contains(obj->GetName())) continue;
2794 if (termfiles.Contains(obj->GetName())) continue;
2795 if (sextra.Contains(obj->GetName())) continue;
2796 if (!termfiles.IsNull()) termfiles += ",";
2797 termfiles += obj->GetName();
2798 }
2799 delete fextra;
2800 }
2801 if (stype.Contains("ter")) {
2802 if (!files.IsNull() && !termfiles.IsNull()) {
2803 files += ",";
2804 files += termfiles;
2805 }
2806 }
2807 return files.Data();
2808}
2809
2810//______________________________________________________________________________
2811Bool_t AliAnalysisAlien::Submit()
2812{
2813// Submit all master jobs.
2814 Int_t nmasterjobs = fInputFiles->GetEntries();
2815 Long_t tshoot = gSystem->Now();
2816 if (!fNsubmitted && !SubmitNext()) return kFALSE;
2817 while (fNsubmitted < nmasterjobs) {
2818 Long_t now = gSystem->Now();
2819 if ((now-tshoot)>30000) {
2820 tshoot = now;
2821 if (!SubmitNext()) return kFALSE;
2822 }
2823 }
2824 return kTRUE;
2825}
2826
2827//______________________________________________________________________________
2828Bool_t AliAnalysisAlien::SubmitMerging()
2829{
2830// Submit all merging jobs.
2831 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
2832 gGrid->Cd(fGridOutputDir);
2833 TString mergeJDLName = fExecutable;
2834 mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2835 if (!fInputFiles) {
2836 Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
2837 return kFALSE;
2838 }
2839 Int_t ntosubmit = fInputFiles->GetEntries();
2840 for (Int_t i=0; i<ntosubmit; i++) {
2841 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
2842 runOutDir.ReplaceAll(".xml", "");
2843 if (fOutputToRunNo) {
2844 // The output directory is the run number
2845 printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
2846 runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
2847 } else {
2848 if (!fRunNumbers.Length() && !fRunRange[0]) {
2849 // The output directory is the grid outdir
2850 printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
2851 runOutDir = fGridOutputDir;
2852 } else {
2853 // The output directory is the master number in 3 digits format
2854 printf("### Submitting merging job for master <%03d>\n", i);
2855 runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
2856 }
2857 }
2858 // Check now the number of merging stages.
2859 TObjArray *list = fOutputFiles.Tokenize(",");
2860 TIter next(list);
2861 TObjString *str;
2862 TString outputFile;
2863 while((str=(TObjString*)next())) {
2864 outputFile = str->GetString();
2865 Int_t index = outputFile.Index("@");
2866 if (index > 0) outputFile.Remove(index);
2867 if (!fMergeExcludes.Contains(outputFile)) break;
2868 }
2869 delete list;
2870 Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
2871 if (!done && (i==ntosubmit-1)) return kFALSE;
2872 if (!fRunNumbers.Length() && !fRunRange[0]) break;
2873 }
2874 if (!ntosubmit) return kTRUE;
2875 Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
2876 \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
2877 \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
2878 \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
2879 \n ################################################################################################################");
2880 gSystem->Exec("aliensh");
2881 return kTRUE;
2882}
2883
2884//______________________________________________________________________________
2885Bool_t AliAnalysisAlien::SubmitNext()
2886{
2887// Submit next bunch of master jobs if the queue is free. The first master job is
2888// submitted right away, while the next will not be unless the previous was split.
2889// The plugin will not submit new master jobs if there are more that 500 jobs in
2890// waiting phase.
2891 static Bool_t iscalled = kFALSE;
2892 static Int_t firstmaster = 0;
2893 static Int_t lastmaster = 0;
2894 static Int_t npermaster = 0;
2895 if (iscalled) return kTRUE;
2896 iscalled = kTRUE;
2897 Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
2898 Int_t ntosubmit = 0;
2899 TGridResult *res;
2900 TString jobID = "";
2901 Int_t nmasterjobs = fInputFiles->GetEntries();
2902 if (!fNsubmitted) {
2903 ntosubmit = 1;
2904 if (!IsUseSubmitPolicy()) {
2905 if (nmasterjobs>5)
2906 Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
2907 \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
2908 ntosubmit = nmasterjobs;
2909 }
2910 } else {
2911 TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
2912 printf("=== master %d: %s\n", lastmaster, status.Data());
2913 // If last master not split, just return
2914 if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
2915 // No more than 100 waiting jobs
2916 if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
2917 npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
2918 if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
2919 if (!ntosubmit) ntosubmit = 1;
2920 printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
2921 nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
2922 }
2923 for (Int_t i=0; i<ntosubmit; i++) {
2924 // Submit for a range of enumeration of runs.
2925 if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
2926 TString query;
2927 TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
2928 runOutDir.ReplaceAll(".xml", "");
2929 if (fOutputToRunNo)
2930 query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
2931 else
2932 query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
2933 printf("********* %s\n",query.Data());
2934 res = gGrid->Command(query);
2935 if (res) {
2936 TString cjobId1 = res->GetKey(0,"jobId");
2937 if (!cjobId1.Length()) {
2938 iscalled = kFALSE;
2939 gGrid->Stdout();
2940 gGrid->Stderr();
2941 Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
2942 return kFALSE;
2943 } else {
2944 Info("StartAnalysis", "\n_______________________________________________________________________ \
2945 \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
2946 \n_______________________________________________________________________",
2947 fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
2948 jobID += cjobId1;
2949 jobID += " ";
2950 lastmaster = cjobId1.Atoi();
2951 if (!firstmaster) firstmaster = lastmaster;
2952 fNsubmitted++;
2953 }
2954 delete res;
2955 } else {
2956 Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
2957 return kFALSE;
2958 }
2959 }
2960 iscalled = kFALSE;
2961 return kTRUE;
2962}
2963
2964//______________________________________________________________________________
2965void AliAnalysisAlien::WriteAnalysisFile()
2966{
2967// Write current analysis manager into the file <analysisFile>
2968 TString analysisFile = fExecutable;
2969 analysisFile.ReplaceAll(".sh", ".root");
2970 if (!TestBit(AliAnalysisGrid::kSubmit)) {
2971 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2972 if (!mgr || !mgr->IsInitialized()) {
2973 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
2974 return;
2975 }
2976 // Check analysis type
2977 TObject *handler;
2978 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
2979 handler = (TObject*)mgr->GetInputEventHandler();
2980 if (handler) {
2981 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
2982 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
2983 }
2984 TDirectory *cdir = gDirectory;
2985 TFile *file = TFile::Open(analysisFile, "RECREATE");
2986 if (file) {
2987 // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
2988 if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
2989 // Unless merging makes no sense
2990 if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
2991 mgr->Write();
2992 delete file;
2993 // Enable termination for local jobs
2994 mgr->SetSkipTerminate(kFALSE);
2995 }
2996 if (cdir) cdir->cd();
2997 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
2998 }
2999 Bool_t copy = kTRUE;
3000 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3001 if (copy) {
3002 CdWork();
3003 TString workdir = gGrid->GetHomeDirectory();
3004 workdir += fGridWorkingDir;
3005 Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
3006 if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
3007 TFile::Cp(Form("file:%s",analysisFile.Data()), Form("alien://%s/%s", workdir.Data(),analysisFile.Data()));
3008 }
3009}
3010
3011//______________________________________________________________________________
3012void AliAnalysisAlien::WriteAnalysisMacro()
3013{
3014// Write the analysis macro that will steer the analysis in grid mode.
3015 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3016 ofstream out;
3017 out.open(fAnalysisMacro.Data(), ios::out);
3018 if (!out.good()) {
3019 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3020 return;
3021 }
3022 Bool_t hasSTEERBase = kFALSE;
3023 Bool_t hasESD = kFALSE;
3024 Bool_t hasAOD = kFALSE;
3025 Bool_t hasANALYSIS = kFALSE;
3026 Bool_t hasANALYSISalice = kFALSE;
3027 Bool_t hasCORRFW = kFALSE;
3028 TString func = fAnalysisMacro;
3029 TString type = "ESD";
3030 TString comment = "// Analysis using ";
3031 if (IsUseMCchain()) {
3032 type = "MC";
3033 comment += "MC";
3034 } else {
3035 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
3036 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
3037 type = "AOD";
3038 comment += "AOD";
3039 }
3040 }
3041 if (type!="AOD" && fFriendChainName!="") {
3042 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
3043 return;
3044 }
3045 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
3046 else comment += " data";
3047 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
3048 func.ReplaceAll(".C", "");
3049 out << "void " << func.Data() << "()" << endl;
3050 out << "{" << endl;
3051 out << comment.Data() << endl;
3052 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
3053 out << " TStopwatch timer;" << endl;
3054 out << " timer.Start();" << endl << endl;
3055 // Change temp directory to current one
3056 out << "// Set temporary merging directory to current one" << endl;
3057 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
3058 // Reset existing include path
3059 out << "// Reset existing include path and add current directory first in the search" << endl;
3060 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
3061 if (!fExecutableCommand.Contains("aliroot")) {
3062 out << "// load base root libraries" << endl;
3063 out << " gSystem->Load(\"libTree\");" << endl;
3064 out << " gSystem->Load(\"libGeom\");" << endl;
3065 out << " gSystem->Load(\"libVMC\");" << endl;
3066 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3067 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3068 }
3069 if (fAdditionalRootLibs.Length()) {
3070 // in principle libtree /lib geom libvmc etc. can go into this list, too
3071 out << "// Add aditional libraries" << endl;
3072 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3073 TIter next(list);
3074 TObjString *str;
3075 while((str=(TObjString*)next())) {
3076 if (str->GetString().Contains(".so"))
3077 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3078 }
3079 if (list) delete list;
3080 }
3081 out << "// Load analysis framework libraries" << endl;
3082 TString setupPar = "AliAnalysisAlien::SetupPar";
3083 if (!fPackages) {
3084 if (!fExecutableCommand.Contains("aliroot")) {
3085 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3086 out << " gSystem->Load(\"libESD\");" << endl;
3087 out << " gSystem->Load(\"libAOD\");" << endl;
3088 }
3089 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3090 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3091 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3092 } else {
3093 TIter next(fPackages);
3094 TObject *obj;
3095 TString pkgname;
3096 while ((obj=next())) {
3097 pkgname = obj->GetName();
3098 if (pkgname == "STEERBase" ||
3099 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3100 if (pkgname == "ESD" ||
3101 pkgname == "ESD.par") hasESD = kTRUE;
3102 if (pkgname == "AOD" ||
3103 pkgname == "AOD.par") hasAOD = kTRUE;
3104 if (pkgname == "ANALYSIS" ||
3105 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3106 if (pkgname == "ANALYSISalice" ||
3107 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3108 if (pkgname == "CORRFW" ||
3109 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3110 }
3111 if (hasANALYSISalice) setupPar = "SetupPar";
3112 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
3113 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
3114 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
3115 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
3116 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
3117 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
3118 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
3119 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
3120 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3121 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
3122 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3123 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
3124 out << "// Compile other par packages" << endl;
3125 next.Reset();
3126 while ((obj=next())) {
3127 pkgname = obj->GetName();
3128 if (pkgname == "STEERBase" ||
3129 pkgname == "STEERBase.par" ||
3130 pkgname == "ESD" ||
3131 pkgname == "ESD.par" ||
3132 pkgname == "AOD" ||
3133 pkgname == "AOD.par" ||
3134 pkgname == "ANALYSIS" ||
3135 pkgname == "ANALYSIS.par" ||
3136 pkgname == "ANALYSISalice" ||
3137 pkgname == "ANALYSISalice.par" ||
3138 pkgname == "CORRFW" ||
3139 pkgname == "CORRFW.par") continue;
3140 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
3141 }
3142 }
3143 out << "// include path" << endl;
3144 // Get the include path from the interpreter and remove entries pointing to AliRoot
3145 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3146 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3147 out << " TIter nextpath(listpaths);" << endl;
3148 out << " TObjString *pname;" << endl;
3149 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3150 out << " TString current = pname->GetName();" << endl;
3151 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3152 out << " gSystem->AddIncludePath(current);" << endl;
3153 out << " }" << endl;
3154 out << " if (listpaths) delete listpaths;" << endl;
3155 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3156 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3157 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
3158 if (fAdditionalLibs.Length()) {
3159 out << "// Add aditional AliRoot libraries" << endl;
3160 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3161 TIter next(list);
3162 TObjString *str;
3163 while((str=(TObjString*)next())) {
3164 if (str->GetString().Contains(".so"))
3165 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3166 if (str->GetString().Contains(".par"))
3167 out << " if (!" << setupPar << "(\"" << str->GetString() << "\")) return;" << endl;
3168 }
3169 if (list) delete list;
3170 }
3171 out << endl;
3172 out << "// analysis source to be compiled at runtime (if any)" << endl;
3173 if (fAnalysisSource.Length()) {
3174 TObjArray *list = fAnalysisSource.Tokenize(" ");
3175 TIter next(list);
3176 TObjString *str;
3177 while((str=(TObjString*)next())) {
3178 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3179 }
3180 if (list) delete list;
3181 }
3182 out << endl;
3183// out << " printf(\"Currently load libraries:\\n\");" << endl;
3184// out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
3185 if (fFastReadOption) {
3186 Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
3187 \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
3188 out << "// fast xrootd reading enabled" << endl;
3189 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
3190 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3191 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
3192 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
3193 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
3194 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3195 }
3196 out << "// connect to AliEn and make the chain" << endl;
3197 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3198 out << "// read the analysis manager from file" << endl;
3199 TString analysisFile = fExecutable;
3200 analysisFile.ReplaceAll(".sh", ".root");
3201 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
3202 out << " if (!file) return;" << endl;
3203 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3204 out << " AliAnalysisManager *mgr = 0;" << endl;
3205 out << " TKey *key;" << endl;
3206 out << " while ((key=(TKey*)nextkey())) {" << endl;
3207 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3208 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3209 out << " };" << endl;
3210 out << " if (!mgr) {" << endl;
3211 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file " << analysisFile <<"\");" << endl;
3212 out << " return;" << endl;
3213 out << " }" << endl << endl;
3214 out << " mgr->PrintStatus();" << endl;
3215 if (AliAnalysisManager::GetAnalysisManager()) {
3216 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3217 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3218 } else {
3219 if (TestBit(AliAnalysisGrid::kTest))
3220 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3221 else
3222 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
3223 }
3224 }
3225 if (IsUsingTags()) {
3226 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
3227 } else {
3228 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
3229 }
3230 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
3231 out << " timer.Stop();" << endl;
3232 out << " timer.Print();" << endl;
3233 out << "}" << endl << endl;
3234 if (IsUsingTags()) {
3235 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
3236 out << "{" << endl;
3237 out << "// Create a chain using tags from the xml file." << endl;
3238 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
3239 out << " if (!coll) {" << endl;
3240 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3241 out << " return NULL;" << endl;
3242 out << " }" << endl;
3243 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
3244 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
3245 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
3246 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
3247 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
3248 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
3249 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
3250 out << " // Check if the cuts configuration file was provided" << endl;
3251 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
3252 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
3253 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3254 out << " }" << endl;
3255 if (fFriendChainName=="") {
3256 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
3257 } else {
3258 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
3259 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
3260 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
3261 }
3262 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
3263 out << " chain->ls();" << endl;
3264 out << " return chain;" << endl;
3265 out << "}" << endl << endl;
3266 if (gSystem->AccessPathName("ConfigureCuts.C")) {
3267 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
3268 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
3269 msg += " AliLHCTagCuts *lhcCuts,\n";
3270 msg += " AliDetectorTagCuts *detCuts,\n";
3271 msg += " AliEventTagCuts *evCuts)";
3272 Info("WriteAnalysisMacro", "%s", msg.Data());
3273 }
3274 }
3275 if (!IsUsingTags() || fFriendChainName!="") {
3276 out <<"//________________________________________________________________________________" << endl;
3277 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
3278 out << "{" << endl;
3279 out << "// Create a chain using url's from xml file" << endl;
3280 out << " TString filename;" << endl;
3281 out << " Int_t run = 0;" << endl;
3282 if (IsUseMCchain()) {
3283 out << " TString treename = \"TE\";" << endl;
3284 } else {
3285 out << " TString treename = type;" << endl;
3286 out << " treename.ToLower();" << endl;
3287 out << " treename += \"Tree\";" << endl;
3288 }
3289 out << " printf(\"***************************************\\n\");" << endl;
3290 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
3291 out << " printf(\"***************************************\\n\");" << endl;
3292 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
3293 out << " if (!coll) {" << endl;
3294 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
3295 out << " return NULL;" << endl;
3296 out << " }" << endl;
3297 out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
3298 out << " TChain *chain = new TChain(treename);" << endl;
3299 if(fFriendChainName!="") {
3300 out << " TChain *chainFriend = new TChain(treename);" << endl;
3301 }
3302 out << " coll->Reset();" << endl;
3303 out << " while (coll->Next()) {" << endl;
3304 out << " filename = coll->GetTURL("");" << endl;
3305 out << " if (mgr) {" << endl;
3306 out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
3307 out << " if (nrun && nrun != run) {" << endl;
3308 out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
3309 out << " mgr->SetRunFromPath(nrun);" << endl;
3310 out << " run = nrun;" << endl;
3311 out << " }" << endl;
3312 out << " }" << endl;
3313 out << " chain->Add(filename);" << endl;
3314 if(fFriendChainName!="") {
3315 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
3316 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3317 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
3318 out << " chainFriend->Add(fileFriend.Data());" << endl;
3319 }
3320 out << " }" << endl;
3321 out << " if (!chain->GetNtrees()) {" << endl;
3322 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
3323 out << " return NULL;" << endl;
3324 out << " }" << endl;
3325 if(fFriendChainName!="") {
3326 out << " chain->AddFriend(chainFriend);" << endl;
3327 }
3328 out << " return chain;" << endl;
3329 out << "}" << endl << endl;
3330 }
3331 if (hasANALYSISalice) {
3332 out <<"//________________________________________________________________________________" << endl;
3333 out << "Bool_t SetupPar(const char *package) {" << endl;
3334 out << "// Compile the package and set it up." << endl;
3335 out << " TString pkgdir = package;" << endl;
3336 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
3337 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
3338 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3339 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3340 out << " // Check for BUILD.sh and execute" << endl;
3341 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3342 out << " printf(\"*******************************\\n\");" << endl;
3343 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3344 out << " printf(\"*******************************\\n\");" << endl;
3345 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3346 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3347 out << " gSystem->ChangeDirectory(cdir);" << endl;
3348 out << " return kFALSE;" << endl;
3349 out << " }" << endl;
3350 out << " } else {" << endl;
3351 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3352 out << " gSystem->ChangeDirectory(cdir);" << endl;
3353 out << " return kFALSE;" << endl;
3354 out << " }" << endl;
3355 out << " // Check for SETUP.C and execute" << endl;
3356 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3357 out << " printf(\"*******************************\\n\");" << endl;
3358 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3359 out << " printf(\"*******************************\\n\");" << endl;
3360 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3361 out << " } else {" << endl;
3362 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3363 out << " gSystem->ChangeDirectory(cdir);" << endl;
3364 out << " return kFALSE;" << endl;
3365 out << " }" << endl;
3366 out << " // Restore original workdir" << endl;
3367 out << " gSystem->ChangeDirectory(cdir);" << endl;
3368 out << " return kTRUE;" << endl;
3369 out << "}" << endl;
3370 }
3371 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
3372 }
3373 Bool_t copy = kTRUE;
3374 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3375 if (copy) {
3376 CdWork();
3377 TString workdir = gGrid->GetHomeDirectory();
3378 workdir += fGridWorkingDir;
3379 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
3380 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
3381 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
3382 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
3383 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
3384 }
3385 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
3386 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
3387 }
3388}
3389
3390//______________________________________________________________________________
3391void AliAnalysisAlien::WriteMergingMacro()
3392{
3393// Write a macro to merge the outputs per master job.
3394 if (!fMergeViaJDL) return;
3395 if (!fOutputFiles.Length()) {
3396 Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3397 return;
3398 }
3399 TString mergingMacro = fExecutable;
3400 mergingMacro.ReplaceAll(".sh","_merge.C");
3401 if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3402 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3403 ofstream out;
3404 out.open(mergingMacro.Data(), ios::out);
3405 if (!out.good()) {
3406 Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
3407 return;
3408 }
3409 Bool_t hasSTEERBase = kFALSE;
3410 Bool_t hasESD = kFALSE;
3411 Bool_t hasAOD = kFALSE;
3412 Bool_t hasANALYSIS = kFALSE;
3413 Bool_t hasANALYSISalice = kFALSE;
3414 Bool_t hasCORRFW = kFALSE;
3415 TString func = mergingMacro;
3416 TString comment;
3417 func.ReplaceAll(".C", "");
3418 out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
3419 out << "{" << endl;
3420 out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
3421 out << " TStopwatch timer;" << endl;
3422 out << " timer.Start();" << endl << endl;
3423 // Reset existing include path
3424 out << "// Reset existing include path and add current directory first in the search" << endl;
3425 out << " gSystem->SetIncludePath(\"-I.\");" << endl;
3426 if (!fExecutableCommand.Contains("aliroot")) {
3427 out << "// load base root libraries" << endl;
3428 out << " gSystem->Load(\"libTree\");" << endl;
3429 out << " gSystem->Load(\"libGeom\");" << endl;
3430 out << " gSystem->Load(\"libVMC\");" << endl;
3431 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
3432 out << " gSystem->Load(\"libMinuit\");" << endl << endl;
3433 }
3434 if (fAdditionalRootLibs.Length()) {
3435 // in principle libtree /lib geom libvmc etc. can go into this list, too
3436 out << "// Add aditional libraries" << endl;
3437 TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
3438 TIter next(list);
3439 TObjString *str;
3440 while((str=(TObjString*)next())) {
3441 if (str->GetString().Contains(".so"))
3442 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3443 }
3444 if (list) delete list;
3445 }
3446 out << "// Load analysis framework libraries" << endl;
3447 if (!fPackages) {
3448 if (!fExecutableCommand.Contains("aliroot")) {
3449 out << " gSystem->Load(\"libSTEERBase\");" << endl;
3450 out << " gSystem->Load(\"libESD\");" << endl;
3451 out << " gSystem->Load(\"libAOD\");" << endl;
3452 }
3453 out << " gSystem->Load(\"libANALYSIS\");" << endl;
3454 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3455 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3456 } else {
3457 TIter next(fPackages);
3458 TObject *obj;
3459 TString pkgname;
3460 TString setupPar = "AliAnalysisAlien::SetupPar";
3461 while ((obj=next())) {
3462 pkgname = obj->GetName();
3463 if (pkgname == "STEERBase" ||
3464 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
3465 if (pkgname == "ESD" ||
3466 pkgname == "ESD.par") hasESD = kTRUE;
3467 if (pkgname == "AOD" ||
3468 pkgname == "AOD.par") hasAOD = kTRUE;
3469 if (pkgname == "ANALYSIS" ||
3470 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
3471 if (pkgname == "ANALYSISalice" ||
3472 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
3473 if (pkgname == "CORRFW" ||
3474 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
3475 }
3476 if (hasANALYSISalice) setupPar = "SetupPar";
3477 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
3478 else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
3479 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
3480 else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
3481 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
3482 else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
3483 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
3484 else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
3485 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
3486 else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
3487 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
3488 else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
3489 out << "// Compile other par packages" << endl;
3490 next.Reset();
3491 while ((obj=next())) {
3492 pkgname = obj->GetName();
3493 if (pkgname == "STEERBase" ||
3494 pkgname == "STEERBase.par" ||
3495 pkgname == "ESD" ||
3496 pkgname == "ESD.par" ||
3497 pkgname == "AOD" ||
3498 pkgname == "AOD.par" ||
3499 pkgname == "ANALYSIS" ||
3500 pkgname == "ANALYSIS.par" ||
3501 pkgname == "ANALYSISalice" ||
3502 pkgname == "ANALYSISalice.par" ||
3503 pkgname == "CORRFW" ||
3504 pkgname == "CORRFW.par") continue;
3505 out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
3506 }
3507 }
3508 out << "// include path" << endl;
3509 // Get the include path from the interpreter and remove entries pointing to AliRoot
3510 out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
3511 out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
3512 out << " TIter nextpath(listpaths);" << endl;
3513 out << " TObjString *pname;" << endl;
3514 out << " while ((pname=(TObjString*)nextpath())) {" << endl;
3515 out << " TString current = pname->GetName();" << endl;
3516 out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
3517 out << " gSystem->AddIncludePath(current);" << endl;
3518 out << " }" << endl;
3519 out << " if (listpaths) delete listpaths;" << endl;
3520 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
3521 out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
3522 out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
3523 if (fAdditionalLibs.Length()) {
3524 out << "// Add aditional AliRoot libraries" << endl;
3525 TObjArray *list = fAdditionalLibs.Tokenize(" ");
3526 TIter next(list);
3527 TObjString *str;
3528 while((str=(TObjString*)next())) {
3529 if (str->GetString().Contains(".so"))
3530 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
3531 }
3532 if (list) delete list;
3533 }
3534 out << endl;
3535 out << "// Analysis source to be compiled at runtime (if any)" << endl;
3536 if (fAnalysisSource.Length()) {
3537 TObjArray *list = fAnalysisSource.Tokenize(" ");
3538 TIter next(list);
3539 TObjString *str;
3540 while((str=(TObjString*)next())) {
3541 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
3542 }
3543 if (list) delete list;
3544 }
3545 out << endl;
3546
3547 if (fFastReadOption) {
3548 Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
3549 out << "// fast xrootd reading enabled" << endl;
3550 out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
3551 out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
3552 out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
3553 out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
3554 out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
3555 out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
3556 }
3557 // Change temp directory to current one
3558 out << "// Set temporary merging directory to current one" << endl;
3559 out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
3560 out << "// Connect to AliEn" << endl;
3561 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
3562 out << " TString outputDir = dir;" << endl;
3563 out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
3564 out << " TString mergeExcludes = \"" << fMergeExcludes << "\";" << endl;
3565 out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
3566 out << " TIter *iter = new TIter(list);" << endl;
3567 out << " TObjString *str;" << endl;
3568 out << " TString outputFile;" << endl;
3569 out << " Bool_t merged = kTRUE;" << endl;
3570 out << " while((str=(TObjString*)iter->Next())) {" << endl;
3571 out << " outputFile = str->GetString();" << endl;
3572 out << " if (outputFile.Contains(\"*\")) continue;" << endl;
3573 out << " Int_t index = outputFile.Index(\"@\");" << endl;
3574 out << " if (index > 0) outputFile.Remove(index);" << endl;
3575 out << " // Skip already merged outputs" << endl;
3576 out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
3577 out << " printf(\"Output file <%s> found. Not merging again.\",outputFile.Data());" << endl;
3578 out << " continue;" << endl;
3579 out << " }" << endl;
3580 out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
3581 out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
3582 out << " if (!merged) {" << endl;
3583 out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
3584 out << " return;" << endl;
3585 out << " }" << endl;
3586 out << " }" << endl;
3587 out << " // all outputs merged, validate" << endl;
3588 out << " ofstream out;" << endl;
3589 out << " out.open(\"outputs_valid\", ios::out);" << endl;
3590 out << " out.close();" << endl;
3591 out << " // read the analysis manager from file" << endl;
3592 TString analysisFile = fExecutable;
3593 analysisFile.ReplaceAll(".sh", ".root");
3594 out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
3595 out << " TFile *file = TFile::Open(\"" << analysisFile << "\");" << endl;
3596 out << " if (!file) return;" << endl;
3597 out << " TIter nextkey(file->GetListOfKeys());" << endl;
3598 out << " AliAnalysisManager *mgr = 0;" << endl;
3599 out << " TKey *key;" << endl;
3600 out << " while ((key=(TKey*)nextkey())) {" << endl;
3601 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
3602 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
3603 out << " };" << endl;
3604 out << " if (!mgr) {" << endl;
3605 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file" << analysisFile <<"\");" << endl;
3606 out << " return;" << endl;
3607 out << " }" << endl << endl;
3608 out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
3609 out << " mgr->SetSkipTerminate(kFALSE);" << endl;
3610 out << " mgr->PrintStatus();" << endl;
3611 if (AliAnalysisManager::GetAnalysisManager()) {
3612 if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
3613 out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
3614 } else {
3615 if (TestBit(AliAnalysisGrid::kTest))
3616 out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
3617 else
3618 out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
3619 }
3620 }
3621 out << " TTree *tree = NULL;" << endl;
3622 out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
3623 out << "}" << endl << endl;
3624 if (hasANALYSISalice) {
3625 out <<"//________________________________________________________________________________" << endl;
3626 out << "Bool_t SetupPar(const char *package) {" << endl;
3627 out << "// Compile the package and set it up." << endl;
3628 out << " TString pkgdir = package;" << endl;
3629 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
3630 out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
3631 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
3632 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
3633 out << " // Check for BUILD.sh and execute" << endl;
3634 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
3635 out << " printf(\"*******************************\\n\");" << endl;
3636 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
3637 out << " printf(\"*******************************\\n\");" << endl;
3638 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
3639 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
3640 out << " gSystem->ChangeDirectory(cdir);" << endl;
3641 out << " return kFALSE;" << endl;
3642 out << " }" << endl;
3643 out << " } else {" << endl;
3644 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
3645 out << " gSystem->ChangeDirectory(cdir);" << endl;
3646 out << " return kFALSE;" << endl;
3647 out << " }" << endl;
3648 out << " // Check for SETUP.C and execute" << endl;
3649 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
3650 out << " printf(\"*******************************\\n\");" << endl;
3651 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
3652 out << " printf(\"*******************************\\n\");" << endl;
3653 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
3654 out << " } else {" << endl;
3655 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
3656 out << " gSystem->ChangeDirectory(cdir);" << endl;
3657 out << " return kFALSE;" << endl;
3658 out << " }" << endl;
3659 out << " // Restore original workdir" << endl;
3660 out << " gSystem->ChangeDirectory(cdir);" << endl;
3661 out << " return kTRUE;" << endl;
3662 out << "}" << endl;
3663 }
3664 }
3665 Bool_t copy = kTRUE;
3666 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3667 if (copy) {
3668 CdWork();
3669 TString workdir = gGrid->GetHomeDirectory();
3670 workdir += fGridWorkingDir;
3671 if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
3672 Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
3673 TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
3674 }
3675}
3676
3677//______________________________________________________________________________
3678Bool_t AliAnalysisAlien::SetupPar(const char *package)
3679{
3680// Compile the par file archive pointed by <package>. This must be present in the current directory.
3681// Note that for loading the compiled library. The current directory should have precedence in
3682// LD_LIBRARY_PATH
3683 TString pkgdir = package;
3684 pkgdir.ReplaceAll(".par","");
3685 gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
3686 TString cdir = gSystem->WorkingDirectory();
3687 gSystem->ChangeDirectory(pkgdir);
3688 // Check for BUILD.sh and execute
3689 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
3690 printf("**************************************************\n");
3691 printf("*** Building PAR archive %s\n", package);
3692 printf("**************************************************\n");
3693 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
3694 ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
3695 gSystem->ChangeDirectory(cdir);
3696 return kFALSE;
3697 }
3698 } else {
3699 ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
3700 gSystem->ChangeDirectory(cdir);
3701 return kFALSE;
3702 }
3703 // Check for SETUP.C and execute
3704 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
3705 printf("**************************************************\n");
3706 printf("*** Setup PAR archive %s\n", package);
3707 printf("**************************************************\n");
3708 gROOT->Macro("PROOF-INF/SETUP.C");
3709 printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
3710 } else {
3711 ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
3712 gSystem->ChangeDirectory(cdir);
3713 return kFALSE;
3714 }
3715 // Restore original workdir
3716 gSystem->ChangeDirectory(cdir);
3717 return kTRUE;
3718}
3719
3720//______________________________________________________________________________
3721void AliAnalysisAlien::WriteExecutable()
3722{
3723// Generate the alien executable script.
3724 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3725 ofstream out;
3726 out.open(fExecutable.Data(), ios::out);
3727 if (out.bad()) {
3728 Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
3729 return;
3730 }
3731 out << "#!/bin/bash" << endl;
3732 // Make sure we can properly compile par files
3733 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
3734 out << "echo \"=========================================\"" << endl;
3735 out << "echo \"############## PATH : ##############\"" << endl;
3736 out << "echo $PATH" << endl;
3737 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3738 out << "echo $LD_LIBRARY_PATH" << endl;
3739 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3740 out << "echo $ROOTSYS" << endl;
3741 out << "echo \"############## which root : ##############\"" << endl;
3742 out << "which root" << endl;
3743 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3744 out << "echo $ALICE_ROOT" << endl;
3745 out << "echo \"############## which aliroot : ##############\"" << endl;
3746 out << "which aliroot" << endl;
3747 out << "echo \"############## system limits : ##############\"" << endl;
3748 out << "ulimit -a" << endl;
3749 out << "echo \"############## memory : ##############\"" << endl;
3750 out << "free -m" << endl;
3751 out << "echo \"=========================================\"" << endl << endl;
3752 out << fExecutableCommand << " ";
3753 out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl << endl;
3754 out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
3755 out << "echo \"############## memory after: ##############\"" << endl;
3756 out << "free -m" << endl;
3757 }
3758 Bool_t copy = kTRUE;
3759 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3760 if (copy) {
3761 CdWork();
3762 TString workdir = gGrid->GetHomeDirectory();
3763 TString bindir = Form("%s/bin", workdir.Data());
3764 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
3765 workdir += fGridWorkingDir;
3766 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
3767 if (FileExists(executable)) gGrid->Rm(executable);
3768 Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
3769 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
3770 }
3771}
3772
3773//______________________________________________________________________________
3774void AliAnalysisAlien::WriteMergeExecutable()
3775{
3776// Generate the alien executable script for the merging job.
3777 if (!fMergeViaJDL) return;
3778 TString mergeExec = fExecutable;
3779 mergeExec.ReplaceAll(".sh", "_merge.sh");
3780 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3781 ofstream out;
3782 out.open(mergeExec.Data(), ios::out);
3783 if (out.bad()) {
3784 Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
3785 return;
3786 }
3787 out << "#!/bin/bash" << endl;
3788 // Make sure we can properly compile par files
3789 out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
3790 out << "echo \"=========================================\"" << endl;
3791 out << "echo \"############## PATH : ##############\"" << endl;
3792 out << "echo $PATH" << endl;
3793 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
3794 out << "echo $LD_LIBRARY_PATH" << endl;
3795 out << "echo \"############## ROOTSYS : ##############\"" << endl;
3796 out << "echo $ROOTSYS" << endl;
3797 out << "echo \"############## which root : ##############\"" << endl;
3798 out << "which root" << endl;
3799 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
3800 out << "echo $ALICE_ROOT" << endl;
3801 out << "echo \"############## which aliroot : ##############\"" << endl;
3802 out << "which aliroot" << endl;
3803 out << "echo \"############## system limits : ##############\"" << endl;
3804 out << "ulimit -a" << endl;
3805 out << "echo \"############## memory : ##############\"" << endl;
3806 out << "free -m" << endl;
3807 out << "echo \"=========================================\"" << endl << endl;
3808 TString mergeMacro = fExecutable;
3809 mergeMacro.ReplaceAll(".sh", "_merge.C");
3810 if (IsOneStageMerging())
3811 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
3812 else
3813 out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
3814 out << fExecutableCommand << " " << "$ARG" << endl;
3815 out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
3816 out << "echo \"############## memory after: ##############\"" << endl;
3817 out << "free -m" << endl;
3818 }
3819 Bool_t copy = kTRUE;
3820 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
3821 if (copy) {
3822 CdWork();
3823 TString workdir = gGrid->GetHomeDirectory();
3824 TString bindir = Form("%s/bin", workdir.Data());
3825 if (!DirectoryExists(bindir)) gGrid->Mkdir(bindir,"-p");
3826 workdir += fGridWorkingDir;
3827 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), mergeExec.Data());
3828 if (FileExists(executable)) gGrid->Rm(executable);
3829 Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
3830 TFile::Cp(Form("file:%s",mergeExec.Data()), Form("alien://%s", executable.Data()));
3831 }
3832}
3833
3834//______________________________________________________________________________
3835void AliAnalysisAlien::WriteProductionFile(const char *filename) const
3836{
3837// Write the production file to be submitted by LPM manager. The format is:
3838// First line: full_path_to_jdl estimated_no_subjobs_per_master
3839// Next lines: full_path_to_dataset XXX (XXX is a string)
3840// To submit, one has to: submit jdl XXX for all lines
3841 ofstream out;
3842 out.open(filename, ios::out);
3843 if (out.bad()) {
3844 Error("WriteProductionFile", "Bad file name: %s", filename);
3845 return;
3846 }
3847 TString workdir;
3848 if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
3849 workdir = gGrid->GetHomeDirectory();
3850 workdir += fGridWorkingDir;
3851 Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
3852 TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
3853 out << locjdl << " " << njobspermaster << endl;
3854 Int_t nmasterjobs = fInputFiles->GetEntries();
3855 for (Int_t i=0; i<nmasterjobs; i++) {
3856 TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3857 runOutDir.ReplaceAll(".xml", "");
3858 if (fOutputToRunNo)
3859 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
3860 else
3861 out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
3862 }
3863 if (gGrid) {
3864 Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
3865 if (FileExists(filename)) gGrid->Rm(filename);
3866 TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
3867 }
3868}
3869
3870//______________________________________________________________________________
3871void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
3872{
3873// Generate the alien validation script.
3874 // Generate the validation script
3875 TObjString *os;
3876 if (fValidationScript.IsNull()) {
3877 fValidationScript = fExecutable;
3878 fValidationScript.ReplaceAll(".sh", "_validation.sh");
3879 }
3880 TString validationScript = fValidationScript;
3881 if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
3882 if (!Connect()) {
3883 Error("WriteValidationScript", "Alien connection required");
3884 return;
3885 }
3886 if (!fTerminateFiles.IsNull()) {
3887 fTerminateFiles.Strip();
3888 fTerminateFiles.ReplaceAll(" ",",");
3889 }
3890 TString outStream = "";
3891 if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
3892 if (!TestBit(AliAnalysisGrid::kSubmit)) {
3893 ofstream out;
3894 out.open(validationScript, ios::out);
3895 out << "#!/bin/bash" << endl;
3896 out << "##################################################" << endl;
3897 out << "validateout=`dirname $0`" << endl;
3898 out << "validatetime=`date`" << endl;
3899 out << "validated=\"0\";" << endl;
3900 out << "error=0" << endl;
3901 out << "if [ -z $validateout ]" << endl;
3902 out << "then" << endl;
3903 out << " validateout=\".\"" << endl;
3904 out << "fi" << endl << endl;
3905 out << "cd $validateout;" << endl;
3906 out << "validateworkdir=`pwd`;" << endl << endl;
3907 out << "echo \"*******************************************************\"" << outStream << endl;
3908 out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
3909 out << "" << endl;
3910 out << "echo \"* Time: $validatetime \"" << outStream << endl;
3911 out << "echo \"* Dir: $validateout\"" << outStream << endl;
3912 out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
3913 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3914 out << "ls -la ./" << outStream << endl;
3915 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
3916 out << "##################################################" << endl;
3917 out << "" << endl;
3918
3919 out << "if [ ! -f stderr ] ; then" << endl;
3920 out << " error=1" << endl;
3921 out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
3922 out << " echo \"Error = $error\" " << outStream << endl;
3923 out << "fi" << endl;
3924
3925 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
3926 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
3927 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
3928 out << "glibcErr=`grep -Ei \"*** glibc detected ***\" stderr`" << endl;
3929 out << "" << endl;
3930
3931 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
3932 out << " error=1" << endl;
3933 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
3934 out << " echo \"$parArch\" " << outStream << endl;
3935 out << " echo \"Error = $error\" " << outStream << endl;
3936 out << "fi" << endl;
3937
3938 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
3939 out << " error=1" << endl;
3940 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
3941 out << " echo \"$segViol\" " << outStream << endl;
3942 out << " echo \"Error = $error\" " << outStream << endl;
3943 out << "fi" << endl;
3944
3945 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
3946 out << " error=1" << endl;
3947 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
3948 out << " echo \"$segFault\" " << outStream << endl;
3949 out << " echo \"Error = $error\" " << outStream << endl;
3950 out << "fi" << endl;
3951
3952 out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
3953 out << " error=1" << endl;
3954 out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
3955 out << " echo \"$glibcErr\" " << outStream << endl;
3956 out << " echo \"Error = $error\" " << outStream << endl;
3957 out << "fi" << endl;
3958
3959 // Part dedicated to the specific analyses running into the train
3960
3961 TString outputFiles = fOutputFiles;
3962 if (merge && !fTerminateFiles.IsNull()) {
3963 outputFiles += ",";
3964 outputFiles += fTerminateFiles;
3965 }
3966 TObjArray *arr = outputFiles.Tokenize(",");
3967 TIter next1(arr);
3968 TString outputFile;
3969 while (!merge && (os=(TObjString*)next1())) {
3970 // No need to validate outputs produced by merging since the merging macro does this
3971 outputFile = os->GetString();
3972 Int_t index = outputFile.Index("@");
3973 if (index > 0) outputFile.Remove(index);
3974 if (fTerminateFiles.Contains(outputFile)) continue;
3975 if (outputFile.Contains("*")) continue;
3976 out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
3977 out << " error=1" << endl;
3978 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
3979 out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
3980 out << "fi" << endl;
3981 }
3982 delete arr;
3983 out << "if ! [ -f outputs_valid ] ; then" << endl;
3984 out << " error=1" << endl;
3985 out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
3986 out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
3987 out << "fi" << endl;
3988
3989 out << "if [ $error = 0 ] ; then" << endl;
3990 out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
3991 if (!IsKeepLogs()) {
3992 out << " echo \"* === Logs std* will be deleted === \"" << endl;
3993 outStream = "";
3994 out << " rm -f std*" << endl;
3995 }
3996 out << "fi" << endl;
3997
3998 out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
3999 out << "echo \"*******************************************************\"" << outStream << endl;
4000 out << "cd -" << endl;
4001 out << "exit $error" << endl;
4002 }
4003 Bool_t copy = kTRUE;
4004 if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4005 if (copy) {
4006 CdWork();
4007 TString workdir = gGrid->GetHomeDirectory();
4008 workdir += fGridWorkingDir;
4009 Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
4010 if (FileExists(validationScript)) gGrid->Rm(validationScript);
4011 TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
4012 }
4013}