1 #ifndef ALIANALYSISALIEN_H
2 #define ALIANALYSISALIEN_H
3 /* Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * See cxx source for full Copyright notice */
6 // Author: Mihaela Gheata, 01/09/2008
8 //==============================================================================
9 // AliAnalysisAlien - AliEn utility class. Provides interface for creating
10 // a personalized JDL, finding and creating a dataset.
11 //==============================================================================
13 #ifndef ALIANALYSISGRID_H
14 #include "AliAnalysisGrid.h"
29 class AliAnalysisManager;
30 class AliAnalysisTaskCfg;
33 class AliAnalysisAlien : public AliAnalysisGrid {
38 AliAnalysisAlien(const char *name);
39 virtual ~AliAnalysisAlien();
40 AliAnalysisAlien(const AliAnalysisAlien& other);
41 AliAnalysisAlien& operator=(const AliAnalysisAlien& other);
43 virtual void AddAdditionalLibrary(const char *name);
44 virtual void AddIncludePath(const char *path);
45 virtual void AddRunNumber(Int_t run);
46 virtual void AddRunNumber(const char *run);
47 virtual void AddRunList(const char *runList);
48 virtual void AddDataFile(const char *lfn);
49 virtual void AddExternalPackage(const char *name);
50 virtual void SetExecutable(const char *name="analysis.sh") {fExecutable = name;}
51 virtual void SetExecutableCommand(const char *command="root -b -q") {fExecutableCommand = command;}
52 virtual void SetArguments(const char *name="") {fArguments = name;}
53 virtual void SetExecutableArgs(const char *name="") {fExecutableArgs = name;}
54 virtual void SetAnalysisMacro(const char *name="myAnalysis.C") {fAnalysisMacro = name;}
55 virtual void SetAnalysisSource(const char *name="myAnalysisClass.cxx") {fAnalysisSource = name;}
56 virtual void SetValidationScript(const char *name="validation.sh") {fValidationScript = name;}
57 virtual void SetAdditionalLibs(const char *list) {fAdditionalLibs = list;}
58 virtual void SetAdditionalRootLibs(const char *list) {fAdditionalRootLibs = list;}
59 virtual void SetPrice(Int_t price=1) {fPrice = price;}
60 virtual void SetRunRange(Int_t min, Int_t max) {fRunRange[0] = min; fRunRange[1] = max;}
61 virtual void SetJobTag(const char *tag="") {fJobTag = tag;}
62 virtual void SetNrunsPerMaster(Int_t nruns=1) {fNrunsPerMaster = nruns;}
63 virtual void SetMaxMergeFiles(Int_t nfiles) {fMaxMergeFiles = nfiles;}
64 virtual void SetMaxMergeStages(Int_t nstages) {fMaxMergeStages = nstages;}
65 virtual void SetSplitMode(const char *type="se") {fSplitMode = type;}
66 virtual void SetSplitMaxInputFileNumber(Int_t nfiles=100) {fSplitMaxInputFileNumber = nfiles;}
67 virtual void SetAPIVersion(const char *version) {fAPIVersion = version;}
68 virtual void SetROOTVersion(const char *version) {fROOTVersion = version;}
69 virtual void SetAliROOTVersion(const char *version) {fAliROOTVersion=version;}
70 virtual void SetUser(const char *user) {fUser = user;}
71 virtual void SetTTL(Int_t ttl=30000) {fTTL = ttl;}
72 virtual void SetGridWorkingDir(const char *name="workdir") {fGridWorkingDir = name;}
73 virtual void SetGridDataDir(const char *name) {fGridDataDir = name;}
74 void SetGeneratorLibs(const char *libs) {fGeneratorLibs = libs;}
75 virtual void SetDataPattern(const char *pattern="*AliESDs.root") {fDataPattern = pattern;}
76 virtual void SetFriendChainName(const char *name="", const char *libnames="");
77 virtual void SetDefaultOutputs(Bool_t flag);
78 virtual void SetGridOutputDir(const char *name="output") {fGridOutputDir = name;}
79 virtual void SetOutputArchive(const char *list="log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2");
80 virtual void SetOutputFiles(const char *list);
81 virtual void SetOutputToRunNo(Int_t mode=1) {fOutputToRunNo = mode;}
82 virtual void SetInputFormat(const char *format="xml-single") {fInputFormat = format;}
83 virtual void SetMaxInitFailed(Int_t nfail=5) {fMaxInitFailed = nfail;}
84 virtual void SetTerminateFiles(const char *list) {fTerminateFiles = list;}
85 virtual void SetMergeExcludes(const char *list) {fMergeExcludes = list; fMergeExcludes.ReplaceAll(",", " "); }
86 virtual void SetMergeViaJDL(Bool_t on=kTRUE) {fMergeViaJDL = on ? 1 : 0;}
87 virtual void SetMergeDirName(const char *name) {fMergeDirName = name;}
88 virtual void SetMasterResubmitThreshold(Int_t percentage) {fMasterResubmitThreshold = percentage;}
89 void SetMCLoop(Bool_t flag=kTRUE) {fMCLoop = flag;}
90 virtual void SetNtestFiles(Int_t nfiles) {fNtestFiles = nfiles;}
91 virtual void SetNumberOfReplicas(Int_t ncopies) {fNreplicas = TMath::Min(ncopies,4);}
92 virtual void SetJDLName(const char *name="analysis.jdl") {fJDLName = name;}
93 virtual void SetPreferedSE(const char *se);
94 virtual void SetProductionMode(Int_t mode=1) {fProductionMode = mode;}
95 virtual void SetRegisterExcludes(const char *list) {fRegisterExcludes = list; fRegisterExcludes.ReplaceAll(",", " "); }
96 virtual void SetRunPrefix(const char *prefix);
97 virtual void SetOutputSingleFolder(const char *folder) {fOutputSingle = folder; fSplitMode="file"; fSplitMaxInputFileNumber=1;}
98 virtual void SetFastReadOption(Bool_t on=kTRUE) {fFastReadOption = on ? 1 : 0;}
99 virtual void SetOverwriteMode(Bool_t on=kTRUE) {fOverwriteMode = on ? 1 : 0;}
100 virtual void SetDropToShell(Bool_t drop=true) {fDropToShell = drop;}
101 virtual void SetTreeName(const char *name) {fTreeName = name;}
103 TGridJDL *GetGridJDL() const {return fGridJDL;}
104 TGridJDL *GetMergingJDL() const {return fMergingJDL;}
105 const char *GetGridOutputDir() const {return fGridOutputDir;}
106 Int_t GetNMCevents() const {return fNMCevents;}
107 Int_t GetNMCjobs() const {return fNMCjobs;}
108 void SetNMCevents(Int_t nevents) {fNMCevents = nevents;}
109 void SetNMCjobs(Int_t njobs) {fNMCjobs = njobs;}
111 void AddModule(AliAnalysisTaskCfg *module);
112 void AddModules(TObjArray *list);
113 AliAnalysisManager *CreateAnalysisManager(const char *name, const char *filename="");
114 Int_t GetNmodules() const;
115 AliAnalysisTaskCfg *GetModule(const char *name);
116 Bool_t LoadModules();
117 Bool_t LoadFriendLibs() const;
118 Bool_t GenerateTest(const char *name, const char *modname="");
119 Bool_t GenerateTrain(const char *name);
120 virtual Bool_t CreateDataset(const char *pattern);
121 Int_t CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output="data.txt", const char *archivefile="", const char *outputdir="data");
122 virtual Bool_t CreateJDL();
123 virtual void EnablePackage(const char *package);
124 static Bool_t DirectoryExists(const char *lfn);
125 static Bool_t FileExists(const char *lfn);
126 static const char *GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone);
127 const char *GetListOfFiles(const char *type);
128 Bool_t CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl="");
129 static AliAnalysisManager *LoadAnalysisManager(const char *fname);
130 static Bool_t MergeInfo(const char *output, const char *collection);
131 static Bool_t MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage=0);
132 virtual Bool_t MergeOutputs();
133 virtual void Print(Option_t *option="") const;
134 static Long64_t RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs);
135 virtual Bool_t StartAnalysis(Long64_t nentries=123456789, Long64_t firstentry=0);
136 static Bool_t SetupPar(const char *package);
137 virtual Bool_t Submit();
138 virtual Bool_t SubmitMerging();
139 static Int_t SubmitSingleJob(const char *query);
140 virtual void WriteAnalysisFile();
141 virtual void WriteAnalysisMacro();
142 virtual void WriteMergingMacro();
143 virtual void WriteMergeExecutable();
144 virtual void WriteExecutable();
145 virtual Bool_t WriteJDL(Bool_t copy);
146 virtual void WriteProductionFile(const char *filename) const;
147 virtual void WriteValidationScript(Bool_t merge=kFALSE);
150 virtual void SetProofCluster(const char *cluster) {fProofCluster = cluster;}
151 virtual void SetProofDataSet(const char *dataset) {fProofDataSet = dataset;}
152 virtual const char *GetProofDataSet() const {return fProofDataSet.Data();}
153 virtual void SetProofParameter(const char *pname, const char *value);
154 const char *GetProofParameter(const char *pname) const;
155 virtual void SetProofReset(Int_t mode) {fProofReset = mode;}
156 virtual void SetNproofWorkers(Int_t nworkers) {fNproofWorkers = nworkers;}
157 virtual void SetNproofWorkersPerSlave(Int_t nworkers) {fNproofWorkersPerSlave = nworkers;}
158 virtual void SetRootVersionForProof(const char *version);
159 virtual void SetAliRootMode(const char *mode) {fAliRootMode = mode;}
160 virtual void SetProofProcessOpt(const char *proofOpt="") {fProofProcessOpt = proofOpt;}
161 virtual TString GetProofProcessOpt() {return fProofProcessOpt;}
162 // .txt file containing the list of files to be chained in test mode
163 virtual void SetFileForTestMode(const char *filename) {fFileForTestMode = filename;}
164 virtual TChain *GetChainForTestMode(const char *treeName) const;
165 virtual const TString& GetGridJobIDs() const { return fGridJobIDs; }
166 virtual const TString& GetGridStages() const { return fGridStages; }
169 Bool_t CheckInputData();
170 void CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags);
171 virtual Bool_t Connect();
172 virtual void SetDefaults();
175 Bool_t IsCollection(const char *lfn) const;
176 Bool_t IsMCLoop() const {return fMCLoop;}
177 virtual Bool_t IsSingleOutput() const;
178 Bool_t IsUsingTags() const {return TObject::TestBit(AliAnalysisGrid::kUseTags);}
179 Bool_t LoadModule(AliAnalysisTaskCfg *mod);
180 Bool_t CheckDependencies();
181 Bool_t CheckFileCopy(const char *alienpath);
184 TGridJDL *fGridJDL; //! JDL maker
185 TGridJDL *fMergingJDL; //! JDL maker
186 Int_t fPrice; // Grid price for the job;
187 Int_t fTTL; // Time to live.
188 Int_t fSplitMaxInputFileNumber; // Maximum number of files to be processed per subjob
189 Int_t fMaxInitFailed; // Maximum initial consecutive subjobs accepted to fail
190 Int_t fMasterResubmitThreshold; // Failed jobs will be resubmitted until this DONE ratio
191 Int_t fNtestFiles; // Number of files used in the testing case
192 Int_t fRunRange[2]; // Run range
193 Int_t fNrunsPerMaster; // Number of runs per masterjob
194 Int_t fMaxMergeFiles; // Maximum number of files to be merged in one chunk
195 Int_t fMaxMergeStages; // Maximum number of merging stages
196 Int_t fNsubmitted; // Number of jobs submitted
197 Int_t fProductionMode; // Production mode (0-off, 1-on)
198 Int_t fOutputToRunNo; // Use run number as output directory
199 Int_t fMergeViaJDL; // Enable merging via automatic JDL
200 Int_t fFastReadOption; // Use xrootd tweaks to reduce timeouts in file access
201 Int_t fOverwriteMode; // Overwrite existing files if any
202 Int_t fNreplicas; // Number of replicas for the output files
203 Int_t fNproofWorkers; // Number of workers in proof mode
204 Int_t fNproofWorkersPerSlave; // Max number of workers per slave in proof mode
205 Int_t fProofReset; // Proof reset mode: 0=no reset, 1=soft, 2=hard
206 Int_t fNMCevents; // Number of MC events in MC loop mode
207 Int_t fNMCjobs; // Number of MC jobs in MC loop mode
208 TString fRunNumbers; // List of runs to be processed
209 TString fExecutable; // Executable script for AliEn job
210 TString fExecutableCommand; // Command(s) to be executed in the executable script
211 TString fArguments; // Arguments for the executable script
212 TString fExecutableArgs; // arguments added to the executable script after the analysis macro
213 TString fAnalysisMacro; // Root macro steering the analysis
214 TString fAnalysisSource; // User analysis implementation (.cxx) file(s)
215 TString fValidationScript; // Name of the validation script
216 TString fAdditionalRootLibs; // List (separated by blacs) of additional libraries needed for/before analysis libs/par file compilation
217 TString fAdditionalLibs; // List (separated by blacs) of additional libraries needed for the analysis loaded AFTER all par files
218 TString fGeneratorLibs; // Extra libraries needed by the generator
219 TString fSplitMode; // Job split mode
220 TString fAPIVersion; // API version
221 TString fROOTVersion; // ROOT version
222 TString fAliROOTVersion; // AliROOT version
223 TString fExternalPackages; // External packages
224 TString fUser; // AliEn user name
225 TString fGridWorkingDir; // AliEn directory containing the input packages
226 TString fGridDataDir; // AliEn data production directory
227 TString fDataPattern; // Data pattern for 'find' command
228 TString fGridOutputDir; // AliEn directory (wrt work dir) where the output should be written
229 TString fOutputArchive; // List of output archives separated by blancs
230 TString fOutputFiles; // List of output files separated by blancs
231 TString fInputFormat; // Input format (xml-single)
232 TString fDatasetName; // Dataset xml file to be created
233 TString fJDLName; // JDL file to be generated
234 TString fTerminateFiles; // List of output files produced during Terminate
235 TString fMergeExcludes; // List of output files excluded from merging
236 TString fRegisterExcludes; // List of liles not to be registered/merged
237 TString fIncludePath; // Include path
238 TString fCloseSE; // Preffered storage element. Taken from alien_CLOSE_SE environment.
239 TString fFriendChainName; // File name to construct friend chain (for AOD)
240 TString fJobTag; // Job tag
241 TString fOutputSingle; // Directory name for the output when split is per file
242 TString fRunPrefix; // Run prefix to be applied to run numbers
243 TString fProofCluster; // Proof cluster name
244 TString fProofDataSet; // Proof dataset to be used
245 TString fFileForTestMode; // .txt file for the chain to be used in PROOF test mode
246 TString fAliRootMode; // AliRoot mode among the list supported by the proof cluster
247 TString fProofProcessOpt; // Option passed to proof process
248 TString fMergeDirName; // Name of the directory that should be added to the output directory
249 TObjArray *fInputFiles; // List of input files to be processed by the job
250 TObjArray *fPackages; // List of packages to be used
251 TObjArray *fModules; // List of AliAnalysisTaskCfg modules
252 TMap fProofParam; // Key-value pairs for proof mode
253 Bool_t fDropToShell; // If true, execute aliensh on start
254 Bool_t fMCLoop; // MC loop flag
255 TString fGridJobIDs; // List of last committed jobs
256 TString fGridStages; // List of last committed jobs
257 TString fFriendLibs; // List of libs (separated by blacs) needed for friends processing
258 TString fTreeName; // Name of the tree to be analyzed
260 ClassDef(AliAnalysisAlien, 26) // Class providing some AliEn utilities