]> git.uio.no Git - u/mrichter/AliRoot.git/blame - PWGLF/FORWARD/trains/ProofHelper.C
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Updates to address coverity issues.
[u/mrichter/AliRoot.git] / PWGLF / FORWARD / trains / ProofHelper.C
CommitLineData
fdfd93b4 1/**
2 * @file ProofHelper.C
3 * @author Christian Holm Christensen <cholm@master.hehi.nbi.dk>
4 * @date Tue Oct 16 18:58:37 2012
5 *
6 * @brief
7 *
8 *
9 * @ingroup pwglf_forward_trains_helper
10 *
11 */
12#ifndef PROOFHELPER_C
13#define PROOFHELPER_C
14#include "Helper.C"
15#ifndef __CINT__
16# include "OutputUtilities.C"
17# include "ParUtilities.C"
18# include "ChainBuilder.C"
19# include <TUrl.h>
20# include <TString.h>
21# include <TProof.h>
22# include <TProofLog.h>
23# include <TProofDebug.h>
24# include <AliAnalysisManager.h>
25# include <TEnv.h>
26# include <TChain.h>
27#else
28class TUrl;
29class TChain;
30#endif
31
32// ===================================================================
33/**
34 * Handle analysis on a Proof farm.
35 *
36 * This helper is triggered by URIs of the form
37 *
38 * @code
39 * proof://[<user>@]<host>[:<port>]/<dsname>[?<options>][#<treename>]
40 * @endcode
41 * where
42 * <dl>
43 * <dt>&lt;user@gt;</dt>
44 * <dd>Optional user name</dd>
45 * <dt>&lt;host@gt;</dt>
46 * <dd>PROOF cluster master host</dd>
47 * <dt>&lt;port@gt;</dt>
48 * <dd>Optional PROOF cluster port on master host</dd>
49 * <dt>&lt;dsname@gt;</dt>
50 * <dd>Data set name</dd>
51 * <dt><tt>&lt;datadir&gt;</tt></dt>
52 * <dd>is the base directory holding data files </dd>
53 * <dt><tt>&lt;collection&gt;</tt></dt>
54 * <dd>is an ASCII or XML list of input sources</dd>
55 * <dt><tt>&lt;file&gt;</tt></dt>
56 * <dd>is a single ROOT file</dd>
57 * <dt>&lt;treename@gt;</dt>
58 * <dd>Optional tree name in data set, often <tt>esdTree</tt> or
59 * <tt>aodTree</tt></dd>
60 * <dt>&lt;options@gt;</dt>
61 * <dd>List of options separated by an &amp;
62 * <dl>
63 * <dt><tt>workers=N[x]</tt></dt>
64 * <dd>Set the number of workers to use. If <tt>x</tt> is appended,
65 * then it's maximum number of workers per slave</dd>
66 * <dt><tt>dsname</tt>[=&lt;output dataset&gt;]</dt>
67 * <dd>Register tree output (e.g., AOD) as a new data set on the
68 * PROOF cluster. If &lt;output dataset&gt; is not specified, take
69 * the name of the train.</dd>
70 * <dt><tt>par[=all]</tt></dt>
71 * <dd>Use PAR files. If the value <tt>all</tt> is given, then also
72 * PAR files of STEERBase, ESD, AOD, ANALYSIS, OADB, ANALYSISalice
73 * are used. </dd>
74 * <dt><tt>mode=[default,rec,sim,train,custom]</tt></dt>
75 * <dd>Set the AliROOT mode. If not specified <tt>default</tt>
76 * is assumed</tt>. See also CreateAliROOTPar</dd>
77 * <dt><tt>storage=&lt;url&gt;</tt></dt>
78 * <dd>Specify a non-default storage location for special output
79 * (e.g., AOD trees). &lt;url&gt; should be a valid XRootd
80 * server URI accessible to the slaves - e.g.,
81 * <tt>root://lxplus.cern.ch:10930//tmp</tt>.</dd>
82 * </dl>
83 * </dd>
84 * </dl>
85 *
86 * @ingroup pwglf_forward_trains_helper
87 */
88struct ProofHelper : public Helper
89{
90 /**
91 * Constructor
92 *
93 * @param url Url
94 * @param opts Options
95 */
96 ProofHelper(const TUrl& url, Int_t verbose)
97 : Helper(url, verbose),
98 fUsePars(false),
99 fBasePars(false)
100 {
101 fOptions.Add("workers", "N[x]", "Number of workers to use", "0");
102 fOptions.Add("dsname", "NAME", "Make output dataset", "");
103 fOptions.Add("par", "tasks|all", "Use par files", "tasks");
104 fOptions.Add("mode", "default|rec|sim", "AliROOT mode", "default");
105 fOptions.Add("storage", "URL", "Location for external storage", "");
106
107 if (!fUrl.GetUser() || fUrl.GetUser()[0] == '\0')
108 fUrl.SetUser(gSystem->GetUserInfo()->fUser);
109 }
110 virtual ~ProofHelper() {}
111 /**
112 * Load a library/PAR/script
113 *
114 * @param name Name
115 * @param par If true, upload & enable PAR
116 * @param slaves If true, also load on slaves
117 *
118 * @return true on success
119 */
120 virtual Bool_t LoadLibrary(const TString& name,
121 Bool_t slaves=true)
122 {
123 if (!fUsePars) {
124 Int_t ret = gSystem->Load(MakeLibraryName(name));
125 if (ret < 0) return false;
126 if (slaves) fExtraLibs.Append(Form(":%s", name.Data()));
127 }
128 else {
129 if (!ParUtilities::Find(name)) {
130 Error("ProofHelper::LoadLibrary", "Failed to find PAR file %s",
131 name.Data());
132 return false;
133 }
134 if (!ParUtilities::Build(name)) {
135 Error("ProofHelper::LoadLibrary", "Failed to build PAR file %s",
136 name.Data());
137 return false;
138 }
139 if (gProof->UploadPackage(name.Data(), TProof::kRemoveOld) < 0) {
140 Error("ProofHelper::LoadLibrary", "Failed to upload PAR file %s",
141 name.Data());
142 return false;
143 }
144 fExtraPars.Append(Form(":%s", name.Data()));
145 }
146 return true;
147 }
148 /**
149 * Load a source file, and compile it
150 *
151 * @param name Name of the source file
152 *
153 * @return true on success
154 */
155 virtual Bool_t LoadSource(const TString& name)
156 {
157 if (!Helper::LoadSource(name)) return false;
158 fExtraSrcs.Append(Form(":%s", gSystem->BaseName(name.Data())));
159 return true;
160 }
161 /**
162 * Set-up to load the AliROOT libraries
163 *
164 * @param par Whether to use PAR files
165 *
166 * @return true on success
167 */
168 virtual Bool_t LoadAliROOT()
169 {
170 if (!gSystem->Getenv("ALICE_ROOT")) {
171 Error("ProofHelper::LoadAliROOT", "Local AliROOT not available");
172 return false;
173 }
174
175 Bool_t tmp = fUsePars;
176 fUsePars = fBasePars;
177 if (!LoadLibrary("STEERBase")) return false;
178 if (!LoadLibrary("ESD")) return false;
179 if (!LoadLibrary("AOD")) return false;
180 if (!LoadLibrary("ANALYSIS")) return false;
181 if (!LoadLibrary("OADB")) return false;
182 if (!LoadLibrary("ANALYSISalice")) return false;
183 fUsePars = tmp;
184
185 return CreateAliROOTPar();
186 }
187 /**
188 * Get the name of the AliROOT par file to use
189 *
190 * @return String
191 */
192 virtual const char* AliROOTParName() const
193 {
194 return "ALIROOT";
195 }
196 /**
197 * Create an AliROOT par file from the executing AliROOT. This PAR
198 * file basically uses the environment of the client - that is, we
199 * assume that the used AliROOT is accessible on the slaves - e.g.,
200 * via an NFS export.
201 *
202 * Note, the SETUP.C script take one argument - a TList of TNamed
203 * parameters. Parameters processed are
204 *
205 * - ALIROOT_MODE=[default,aliroot,rec,sim,train]
206 * - default: Load base analysis libraries
207 * - aliroot: Load $ALICE_ROOT/macros/loadlibs.C
208 * - rec: Load $ALICE_ROOT/macros/loadlibsrec.C
209 * - sim: Load $ALICE_ROOT/macros/loadlibssim.C
210 * - ALIROOT_EXTRA_LIBS Colon separated list of additional (Ali)ROOT
211 * libraries to load on the slaves.
212 *
213 * The generated PAR file is uploaded but not enabled until we have
214 * populated fExtraLibs. The enabling takes place at the end of the
215 * set-up.
216 *
217 * @return true on success, false otherwise. */
218 virtual Bool_t CreateAliROOTPar()
219 {
220 if (fBasePars) return true;
221
222 TString parName(AliROOTParName());
223 // Set-up directories
224 if (gSystem->MakeDirectory(parName) < 0) {
225 Error("ProofHelper::CreateAliROOTPar", "Could not make directory '%s'",
226 parName.Data());
227 return false;
228 }
229
230 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", parName.Data()))) {
231 Error("ProofHelper::CreateAliROOTPar",
232 "Could not make directory %s/PROOF-INF",
233 parName.Data());
234 return false;
235 }
236
237 std::ofstream b(Form("%s/PROOF-INF/BUILD.sh",parName.Data()));
238 if (!b) {
239 Error("ProofHelper::CreateAliROOTPar",
240 "Failed to make BUILD.sh shell script");
241 return false;
242 }
243 b << "#!/bin/sh\n\n"
244 << "# echo Nothing to do\n"
245 << "exit 0\n"
246 << std::endl;
247 b.close();
248 gSystem->Exec(Form("chmod a+x %s/PROOF-INF/BUILD.sh",parName.Data()));
249
250 std::ofstream s(Form("%s/PROOF-INF/SETUP.C", parName.Data()));
251 if (!s) {
252 Error("ProofHelper::CreateAliROOTPar",
253 "Failed to make SETUP.C ROOT script");
254 return false;
255 }
256 s << "void SETUP(TList* opts) {\n"
257 << " gSystem->Setenv(\"ALICE\",\""
258 << gSystem->Getenv("ALICE") << "\");\n"
259 << " gSystem->Setenv(\"ALICE_ROOT\",\""
260 << gSystem->Getenv("ALICE_ROOT") << "\");\n"
261 << " gSystem->Setenv(\"ALICE_TARGET\",\""
262 << gSystem->Getenv("ALICE_TARGET") << "\");\n"
263 << " gSystem->AddDynamicPath("
264 << "\"$(ALICE_ROOT)/lib/tgt_$(ALICE_TARGET)\");\n";
265 if (gSystem->Getenv("OADB_PATH"))
266 s << " gSystem->Setenv(\"OADB_PATH\",\""
267 << gSystem->Getenv("OADB_PATH") << "\");\n";
268 s << " \n"
269 << " // Info(\"SETUP\",\"Loading ROOT libraries\");\n"
270 << " gSystem->Load(\"libTree\");\n"
271 << " gSystem->Load(\"libGeom\");\n"
272 << " gSystem->Load(\"libVMC\");\n"
273 << " gSystem->Load(\"libPhysics\");\n"
274 << " gSystem->Load(\"libMinuit\");\n"
275 << " \n";
276 s << " // Info(\"SETUP\",\"Parameter list:\");\n"
277 << " if (!opts) return;\n"
278 << " //opts->ls();\n"
279 << " \n";
280 s << " TObject* par = opts->FindObject(\"ALIROOT_MODE\");\n"
281 << " if (par) {\n"
282 << " // Info(\"SETUP\",\"ALIROOT mode: %s\", par->GetTitle());\n"
283 << " TString mode(par->GetTitle());\n"
284 << " if (mode.EqualTo(\"default\",TString::kIgnoreCase)) {\n"
285 << " gSystem->Load(\"libSTEERBase\");\n"
286 << " gSystem->Load(\"libESD\");\n"
287 << " gSystem->Load(\"libAOD\");\n"
288 << " gSystem->Load(\"libANALYSIS\");\n"
289 << " gSystem->Load(\"libOADB\");\n"
290 << " gSystem->Load(\"libANALYSISalice\");\n"
291 << " }\n"
292 << " else if (mode.EqualTo(\"aliroot\",TString::kIgnoreCase)) \n"
293 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibs.C\");\n"
294 << " else if (mode.EqualTo(\"rec\",TString::kIgnoreCase)) \n"
295 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibsrec.C\");\n"
296 << " else if (mode.EqualTo(\"sim\",TString::kIgnoreCase)) \n"
297 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibssim.C\");\n"
298 << " else if (mode.EqualTo(\"train\",TString::kIgnoreCase)) \n"
299 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibstrain.C\");\n"
300 << " else if (mode.EqualTo(\"custom\",TString::kIgnoreCase)) \n"
301 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibstrain.C\");\n"
302 << " }\n"
303 << " \n";
304 s << " par = opts->FindObject(\"ALIROOT_EXTRA_LIBS\");\n"
305 << " if (par) {\n"
306 << " Info(\"SETUP\",\"Libaries to load: %s\n\",par->GetTitle());\n"
307 << " TString tit(par->GetTitle());\n"
308 << " TObjArray* tokens = tit.Tokenize(\":\");\n"
309 << " TObject* lib = 0;\n"
310 << " TIter next(tokens);\n"
311 << " while ((lib = next())) {\n"
312 << " TString libName(lib->GetName());\n"
313 << " if (!libName.BeginsWith(\"lib\")) libName.Prepend(\"lib\");\n"
314 << " // Info(\"SETUP\",\"Loading %s ...\",libName.Data());\n"
315 << " gSystem->Load(Form(\"lib%s\",lib->GetName()));\n"
316 << " }\n"
317 << " }\n"
318 << "}\n"
319 << std::endl;
320 s.close();
321
322 Int_t ret = gSystem->Exec(Form("tar -czf %s.par %s",
323 parName.Data(), parName.Data()));
324 if (ret != 0) {
325 Error("ProofHelper::CreateAliROOTPar", "Failed to pack up PAR files");
326 return false;
327 }
328
329 ret = gProof->UploadPackage(Form("./%s.par", parName.Data()),
330 TProof::kRemoveOld);
331 if (ret != 0) {
332 Error("ProofHelper::CreateAliROOTPar",
333 "Failed to upload the AliROOT PAR file");
334 return false;
335 }
336 // Note, the PAR isn't enabled until much later when we've
337 // collected all the needed libraries in fExtraLibs
338 return true;
339 }
340 /**
341 * Get the mode identifier
342 *
343 * @return Always kProof
344 */
345 virtual UShort_t Mode() const { return kProof; }
346 /**
347 * Get the mode string used for AliAnalysisManager::StartAnalysis
348 */
349 virtual const char* ModeString() const { return "proof"; }
350 /**
351 * Set-up done before task set-ups
352 *
353 * @return true on success
354 */
355 virtual Bool_t PreSetup()
356 {
357 // --- Set prefered GSI method ---------------------------------
358 gEnv->SetValue("XSec.GSI.DelegProxy", "2");
359
360 // --- Add ALICE_ROOT directory to search path for packages ----
361 Info("ProofHelper::PreSetup", "Set location of packages");
362 gEnv->SetValue("Proof.GlobalPackageDirs",
363 Form("%s:%s",
364 gEnv->GetValue("Proof.GlobalPackageDirs", "."),
365 gSystem->Getenv("ALICE_ROOT")));
366
367 // --- PAR parameters --------------------------------------------
368 fUsePars = fOptions.Has("par");
369 fBasePars = (fUsePars &&
370 fOptions.Get("par").EqualTo("all",TString::kIgnoreCase));
371
372 // --- Connect to the cluster ------------------------------------
373 TUrl connect(fUrl);
374 connect.SetAnchor("");
375 connect.SetFile("");
376 connect.SetOptions("");
377 TString opts;
378 if (fOptions.Has("workers"))
379 opts.Append(Form("workers=%s", fOptions.Get("workers").Data()));
380
381 Info("ProofHelper::PreSetup", "Connecting to %s with %soptions %s",
382 connect.GetUrl(),
383 opts.IsNull() ? "no " : "",
384 opts.Data());
385 TString proto(connect.GetProtocol());
386 if (proto.BeginsWith("lite") && fOptions.Has("workers"))
387 TProof::Open(opts);
388 else
389 TProof::Open(connect.GetUrl(), opts);
390 // TProof::Open(connect.GetHost(), opts);
391 if (!gProof) {
392 Error("ProofHelper::PreSetup", "Failed to open Proof connection %s",
393 connect.GetUrl());
394 return false;
395 }
396 Info("ProofHelper::PreSetup", "Using progress dialog=%d",
397 gProof->TestBit(TProof::kUseProgressDialog));
398 return true;
399 }
400 /**
401 * Set-up done after the task set-ups
402 *
403 * @return true on success
404 */
405 virtual Bool_t PostSetup()
406 {
407 AliAnalysisManager* mgr = AliAnalysisManager::GetAnalysisManager();
408 if (!mgr) {
409 Error("ProofHelper::PostSetup", "No analysis manager defined");
410 return false;
411 }
412
413 // --- Check for output ------------------------------------------
414 if (fOptions.Has("dsname"))
415 OutputUtilities::RegisterDataset(fOptions.Get("dsname"));
416 if (fOptions.Has("storage"))
417 OutputUtilities::RegisterStorage(fOptions.Get("storage"));
418
419 // --- If we are not using PARs for Base, enable special PAR -----
420 if (!fBasePars) {
421 TString tmp(fExtraLibs.Strip(TString::kBoth,':'));
422 TList* params = new TList;
423 params->SetOwner(true);
424 params->Add(new TNamed("ALIROOT_EXTRA_LIBS", tmp.Data()));
425 if (fOptions.Has("mode"))
426 params->Add(new TNamed("ALIROOT_MODE", fOptions.Get("mode").Data()));
427 else
428 params->Add(new TNamed("ALIROOT_MODE", "default"));
429 Int_t ret = gProof->EnablePackage(AliROOTParName(), params, true);
430 if (ret < 0) {
431 Error("ProofHelper::EnableAliROOT", "Failed to enable AliROOT PAR %s",
432 AliROOTParName());
433 return false;
434 }
435 }
436
437 // --- Load par files --------------------------------------------
438 TString tmp = fExtraPars.Strip(TString::kBoth,':');
439 TObjArray* pars = tmp.Tokenize(":");
440 TObject* obj = 0;
441 TIter next(pars);
442 while ((obj = next())) {
443 Int_t ret = gProof->EnablePackage(obj->GetName());
444 if (ret < 0) {
445 Error("ProofHelper::PostSetup", "Failed to enable PAR %s",
446 obj->GetName());
447 return false;
448 }
449 }
450
451 // --- Load extra sources ----------------------------------------
452 TString tmp2 = fExtraSrcs.Strip(TString::kBoth, ':');
453 TObjArray* srcs = tmp2.Tokenize(":");
454 TIter next2(srcs);
455 while ((obj = next())) {
456 Int_t ret = gProof->Load(Form("%s++g", obj->GetName()), true);
457 if (ret < 0) {
458 Error("ProofHelper::PostSetup", "Failed to compile %s", obj->GetName());
459 return false;
460 }
461 }
462 return true;
463 }
464 /**
465 * Start the analysis
466 *
467 * @param nEvents Number of events to analyse
468 *
469 * @return The return value of AliAnalysisManager::StartAnalysis
470 */
471 virtual Long64_t Run(Long64_t nEvents=-1)
472 {
473 AliAnalysisManager* mgr = AliAnalysisManager::GetAnalysisManager();
474 gProof->SetLogLevel(TMath::Max(fVerbose-2,0),
475 /* TProofDebug::kPacketizer| */
476 TProofDebug::kLoop|
477 /* TProofDebug::kSelector|
478 TProofDebug::kOutput|
479 TProofDebug::kInput|
480 TProofDebug::kGlobal|*/
481 TProofDebug::kPackage);
482 TString dsName(fUrl.GetFile());
483 // if (fUrl.GetAnchor() && fUrl.GetAnchor()[0] != '\0')
484 // dsName.Append(Form("#%s", fUrl.GetAnchor()));
485 Long64_t ret = mgr->StartAnalysis(fUrl.GetProtocol(), dsName, nEvents);
486
487 if (fVerbose > 2)
488 TProof::Mgr(fUrl.GetUrl())->GetSessionLogs()->Save("*","proof.log");
489 return ret;
490 }
491 /**
492 * Print information to standard output
493 *
494 * @param option
495 */
496 virtual void Print(Option_t* option="") const
497 {
498 Helper::Print(option);
499 std::cout << std::boolalpha
500 << " --- Other settings -------\n"
501 << " Extra libraries : " << fExtraLibs << "\n"
502 << " Extra PARs : " << fExtraPars << "\n"
503 << " Extra sources : " << fExtraSrcs << "\n"
504 << " Use PARs of tasks: " << fUsePars << "\n"
505 << " Use PARs of base : " << fBasePars
506 << std::noboolalpha << std::endl;
507 }
508 /**
509 * Path of output
510 *
511 * @return Path to output - possibly a data set
512 */
513 virtual TString OutputPath() const
514 {
515 TString ret;
516 if (fOptions.Has("dsname")) {
517 ret = Form("/%s/%s/", gProof->GetGroup(), gProof->GetUser());
518 ret.Append(OutputUtilities::RegisteredDataset());
519 }
520 return ret;
521 }
522 /**
523 * @return URL help string
524 */
525 virtual const Char_t* UrlHelp() const
526 {
527 return "proof://<host>[:<port>]/[<dataset>|<path>][?<options>][#<treeName>]";
528 }
529 /**
530 * @return Short description
531 */
532 virtual const char* Desc() const { return "PROOF"; }
533 TString fExtraLibs;
534 TString fExtraPars;
535 TString fExtraSrcs;
536 Bool_t fUsePars;
537 Bool_t fBasePars;
538};
539#endif
540//
541// EOF
542//