3 * @author Christian Holm Christensen <cholm@master.hehi.nbi.dk>
4 * @date Tue Oct 16 18:58:37 2012
9 * @ingroup pwglf_forward_trains_helper
16 # include "OutputUtilities.C"
17 # include "ParUtilities.C"
18 # include "ChainBuilder.C"
22 # include <TProofLog.h>
23 # include <TProofDebug.h>
24 # include <AliAnalysisManager.h>
32 // ===================================================================
34 * Handle analysis on a Proof farm.
36 * This helper is triggered by URIs of the form
39 * proof://[<user>@]<host>[:<port>]/<dsname>[?<options>][#<treename>]
43 * <dt><user@gt;</dt>
44 * <dd>Optional user name</dd>
45 * <dt><host@gt;</dt>
46 * <dd>PROOF cluster master host</dd>
47 * <dt><port@gt;</dt>
48 * <dd>Optional PROOF cluster port on master host</dd>
49 * <dt><dsname@gt;</dt>
50 * <dd>Data set name</dd>
51 * <dt><tt><datadir></tt></dt>
52 * <dd>is the base directory holding data files </dd>
53 * <dt><tt><collection></tt></dt>
54 * <dd>is an ASCII or XML list of input sources</dd>
55 * <dt><tt><file></tt></dt>
56 * <dd>is a single ROOT file</dd>
57 * <dt><treename@gt;</dt>
58 * <dd>Optional tree name in data set, often <tt>esdTree</tt> or
59 * <tt>aodTree</tt></dd>
60 * <dt><options@gt;</dt>
61 * <dd>List of options separated by an &
63 * <dt><tt>workers=N[x]</tt></dt>
64 * <dd>Set the number of workers to use. If <tt>x</tt> is appended,
65 * then it's maximum number of workers per slave</dd>
66 * <dt><tt>dsname</tt>[=<output dataset>]</dt>
67 * <dd>Register tree output (e.g., AOD) as a new data set on the
68 * PROOF cluster. If <output dataset> is not specified, take
69 * the name of the train.</dd>
70 * <dt><tt>par[=all]</tt></dt>
71 * <dd>Use PAR files. If the value <tt>all</tt> is given, then also
72 * PAR files of STEERBase, ESD, AOD, ANALYSIS, OADB, ANALYSISalice
74 * <dt><tt>mode=[default,rec,sim,train,custom]</tt></dt>
75 * <dd>Set the AliROOT mode. If not specified <tt>default</tt>
76 * is assumed</tt>. See also CreateAliROOTPar</dd>
77 * <dt><tt>storage=<url></tt></dt>
78 * <dd>Specify a non-default storage location for special output
79 * (e.g., AOD trees). <url> should be a valid XRootd
80 * server URI accessible to the slaves - e.g.,
81 * <tt>root://lxplus.cern.ch:10930//tmp</tt>.</dd>
86 * @ingroup pwglf_forward_trains_helper
88 struct ProofHelper : public Helper
96 ProofHelper(const TUrl& url, Int_t verbose)
97 : Helper(url, verbose),
104 fOptions.Add("workers", "N[x]", "Number of workers to use", "0");
105 fOptions.Add("dsname", "NAME", "Make output dataset", "");
106 fOptions.Add("par", "tasks|all", "Use par files", "tasks");
107 fOptions.Add("mode", "default|rec|sim", "AliROOT mode", "default");
108 fOptions.Add("storage", "URL", "Location for external storage", "");
109 fOptions.Add("wrapper", "CMD", "Wrapper command", "");
110 fOptions.Add("clear", "PKGS", "Clear packages ','-separated", "");
111 fOptions.Add("reset", "soft|hard", "Reset cluster", "hard");
112 if (!fUrl.GetUser() || fUrl.GetUser()[0] == '\0')
113 fUrl.SetUser(gSystem->GetUserInfo()->fUser);
118 virtual ~ProofHelper() {}
120 * Load a library/PAR/script
123 * @param par If true, upload & enable PAR
124 * @param slaves If true, also load on slaves
126 * @return true on success
128 virtual Bool_t LoadLibrary(const TString& name,
132 Int_t ret = gSystem->Load(MakeLibraryName(name));
133 if (ret < 0) return false;
134 if (slaves) fExtraLibs.Append(Form(":%s", name.Data()));
137 if (!ParUtilities::Find(name)) {
138 Error("ProofHelper::LoadLibrary", "Failed to find PAR file %s",
142 if (!ParUtilities::Build(name)) {
143 Error("ProofHelper::LoadLibrary", "Failed to build PAR file %s",
147 if (gProof->UploadPackage(name.Data(), TProof::kRemoveOld) < 0) {
148 Error("ProofHelper::LoadLibrary", "Failed to upload PAR file %s",
152 fExtraPars.Append(Form(":%s", name.Data()));
157 * Load a source file, and compile it
159 * @param name Name of the source file
161 * @return true on success
163 virtual Bool_t LoadSource(const TString& name)
165 if (!Helper::LoadSource(name)) return false;
166 fExtraSrcs.Append(Form(":%s", gSystem->BaseName(name.Data())));
170 * Set-up to load the AliROOT libraries
172 * @param par Whether to use PAR files
174 * @return true on success
176 virtual Bool_t LoadAliROOT()
178 if (!gSystem->Getenv("ALICE_ROOT")) {
179 Error("ProofHelper::LoadAliROOT", "Local AliROOT not available");
183 Bool_t tmp = fUsePars;
184 fUsePars = fBasePars;
185 if (!LoadLibrary("STEERBase")) return false;
186 if (!LoadLibrary("ESD")) return false;
187 if (!LoadLibrary("AOD")) return false;
188 if (!LoadLibrary("ANALYSIS")) return false;
189 if (!LoadLibrary("OADB")) return false;
190 if (!LoadLibrary("ANALYSISalice")) return false;
193 return CreateAliROOTPar();
196 * Get the name of the AliROOT par file to use
200 virtual const char* AliROOTParName() const
205 * Create an AliROOT par file from the executing AliROOT. This PAR
206 * file basically uses the environment of the client - that is, we
207 * assume that the used AliROOT is accessible on the slaves - e.g.,
210 * Note, the SETUP.C script take one argument - a TList of TNamed
211 * parameters. Parameters processed are
213 * - ALIROOT_MODE=[default,aliroot,rec,sim,train]
214 * - default: Load base analysis libraries
215 * - aliroot: Load $ALICE_ROOT/macros/loadlibs.C
216 * - rec: Load $ALICE_ROOT/macros/loadlibsrec.C
217 * - sim: Load $ALICE_ROOT/macros/loadlibssim.C
218 * - ALIROOT_EXTRA_LIBS Colon separated list of additional (Ali)ROOT
219 * libraries to load on the slaves.
221 * The generated PAR file is uploaded but not enabled until we have
222 * populated fExtraLibs. The enabling takes place at the end of the
225 * @return true on success, false otherwise. */
226 virtual Bool_t CreateAliROOTPar()
228 if (fBasePars) return true;
230 TString parName(AliROOTParName());
231 TString parFile(Form("%s.par", parName.Data()));
233 // --- Check if we have the drirectory already -------------------
234 if (gSystem->AccessPathName(parName.Data()) == 0) {
235 // Let's remove it to get a clean slate
236 if (gSystem->Exec(Form("rm -rf %s", parName.Data())) != 0) {
237 Error("ProofHelper", "Failed to remove %s", parName.Data());
241 // --- Check if the PAR file is there, and remove it if so -------
242 if (gSystem->AccessPathName(parFile.Data()) == 0) {
243 if (gSystem->Unlink(parFile.Data()) != 0) {
244 Error("ProofHelper::CreateAliROOTPar", "Failed to remove %s",
251 // Set-up directories
252 if (gSystem->MakeDirectory(parName) < 0) {
253 Error("ProofHelper::CreateAliROOTPar", "Could not make directory '%s'",
258 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", parName.Data()))) {
259 Error("ProofHelper::CreateAliROOTPar",
260 "Could not make directory %s/PROOF-INF",
265 std::ofstream b(Form("%s/PROOF-INF/BUILD.sh",parName.Data()));
267 Error("ProofHelper::CreateAliROOTPar",
268 "Failed to make BUILD.sh shell script");
272 << "# echo Nothing to do\n"
276 gSystem->Exec(Form("chmod a+x %s/PROOF-INF/BUILD.sh",parName.Data()));
278 std::ofstream s(Form("%s/PROOF-INF/SETUP.C", parName.Data()));
280 Error("ProofHelper::CreateAliROOTPar",
281 "Failed to make SETUP.C ROOT script");
284 s << "void SETUP(TList* opts) {\n"
285 << " gSystem->Setenv(\"ALICE\",\""
286 << gSystem->Getenv("ALICE") << "\");\n"
287 << " gSystem->Setenv(\"ALICE_ROOT\",\""
288 << gSystem->Getenv("ALICE_ROOT") << "\");\n"
289 << " gSystem->Setenv(\"ALICE_TARGET\",\""
290 << gSystem->Getenv("ALICE_TARGET") << "\");\n"
291 << " gSystem->AddDynamicPath("
292 << "\"$(ALICE_ROOT)/lib/tgt_$(ALICE_TARGET)\");\n";
293 if (gSystem->Getenv("OADB_PATH"))
294 s << " gSystem->Setenv(\"OADB_PATH\",\""
295 << gSystem->Getenv("OADB_PATH") << "\");\n";
297 << " // Info(\"SETUP\",\"Loading ROOT libraries\");\n"
298 << " gSystem->Load(\"libTree\");\n"
299 << " gSystem->Load(\"libGeom\");\n"
300 << " gSystem->Load(\"libVMC\");\n"
301 << " gSystem->Load(\"libPhysics\");\n"
302 << " gSystem->Load(\"libMinuit\");\n"
304 s << " // Info(\"SETUP\",\"Parameter list:\");\n"
305 << " if (!opts) return;\n"
306 << " //opts->ls();\n"
308 s << " TObject* par = opts->FindObject(\"ALIROOT_MODE\");\n"
310 << " // Info(\"SETUP\",\"ALIROOT mode: %s\", par->GetTitle());\n"
311 << " TString mode(par->GetTitle());\n"
312 << " if (mode.EqualTo(\"default\",TString::kIgnoreCase)) {\n"
313 << " gSystem->Load(\"libSTEERBase\");\n"
314 << " gSystem->Load(\"libESD\");\n"
315 << " gSystem->Load(\"libAOD\");\n"
316 << " gSystem->Load(\"libANALYSIS\");\n"
317 << " gSystem->Load(\"libOADB\");\n"
318 << " gSystem->Load(\"libANALYSISalice\");\n"
320 << " else if (mode.EqualTo(\"aliroot\",TString::kIgnoreCase)) \n"
321 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibs.C\");\n"
322 << " else if (mode.EqualTo(\"rec\",TString::kIgnoreCase)) \n"
323 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibsrec.C\");\n"
324 << " else if (mode.EqualTo(\"sim\",TString::kIgnoreCase)) \n"
325 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibssim.C\");\n"
326 << " else if (mode.EqualTo(\"train\",TString::kIgnoreCase)) \n"
327 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibstrain.C\");\n"
328 << " else if (mode.EqualTo(\"custom\",TString::kIgnoreCase)) \n"
329 << " gROOT->Macro(\"$ALICE_ROOT/macros/loadlibstrain.C\");\n"
332 s << " par = opts->FindObject(\"ALIROOT_EXTRA_LIBS\");\n"
334 << " Info(\"SETUP\",\"Libaries to load: %s\n\",par->GetTitle());\n"
335 << " TString tit(par->GetTitle());\n"
336 << " TObjArray* tokens = tit.Tokenize(\":\");\n"
337 << " TObject* lib = 0;\n"
338 << " TIter next(tokens);\n"
339 << " while ((lib = next())) {\n"
340 << " TString libName(lib->GetName());\n"
341 << " if (!libName.BeginsWith(\"lib\")) libName.Prepend(\"lib\");\n"
342 << " // Info(\"SETUP\",\"Loading %s ...\",libName.Data());\n"
343 << " gSystem->Load(Form(\"lib%s\",lib->GetName()));\n"
350 Int_t ret = gSystem->Exec(Form("tar -czf %s %s",
351 parFile.Data(), parName.Data()));
353 Error("ProofHelper::CreateAliROOTPar", "Failed to pack up PAR file %s",
358 ret = gProof->UploadPackage(parFile.Data(),TProof::kRemoveOld);
360 Error("ProofHelper::CreateAliROOTPar",
361 "Failed to upload the AliROOT PAR file");
364 // Note, the PAR isn't enabled until much later when we've
365 // collected all the needed libraries in fExtraLibs
369 * Get the mode identifier
371 * @return Always kProof
373 virtual UShort_t Mode() const { return kProof; }
375 * Get the mode string used for AliAnalysisManager::StartAnalysis
377 virtual const char* ModeString() const { return "proof"; }
379 * Set-up done before task set-ups
381 * @return true on success
383 virtual Bool_t PreSetup()
385 // --- Set prefered GSI method ---------------------------------
386 gEnv->SetValue("XSec.GSI.DelegProxy", "2");
388 // --- Add ALICE_ROOT directory to search path for packages ----
389 // Info("ProofHelper::PreSetup", "Set location of packages");
390 gEnv->SetValue("Proof.GlobalPackageDirs",
392 gEnv->GetValue("Proof.GlobalPackageDirs", "."),
393 gSystem->Getenv("ALICE_ROOT")));
395 // --- Forming the URI we use to connect with --------------------
397 connect.SetAnchor("");
399 connect.SetOptions("");
401 // --- Check if we need to reset first ---------------------------
402 if (fOptions.Has("reset")) {
403 TString reset = fOptions.Get("reset");
404 Bool_t hard = (reset.IsNull() ||
405 reset.EqualTo("hard", TString::kIgnoreCase));
406 Info("ProofHelper::PreSetup", "Doing a %s reset of %s",
407 hard ? "hard" : "soft", connect.GetUrl());
408 TProof::Reset(connect.GetUrl(), hard);
410 Info("ProofHelper::PreSetup",
411 "Waiting for %d second%s for things to settle", secs,
412 secs > 1 ? "s" : "");
413 gSystem->Sleep(1000*secs);
416 // --- Check if we're using a wrapper ----------------------------
417 if (fOptions.Has("wrapper")) {
418 TString wrapper = fOptions.Get("wrapper");
419 if (wrapper.IsNull())
420 // In case of no argument, use GDB
421 // Just run and backtrace
422 wrapper = "/usr/bin/gdb --batch -ex run -ex bt --args";
423 Info("ProofHelper::PreSetup", "Using wrapper command: %s",
425 TProof::AddEnvVar("PROOF_WRAPPERCMD", wrapper);
428 // --- PAR parameters --------------------------------------------
429 fUsePars = fOptions.Has("par");
430 fBasePars = (fUsePars &&
431 fOptions.Get("par").EqualTo("all",TString::kIgnoreCase));
433 // --- Connect to the cluster ------------------------------------
435 if (fOptions.Has("workers"))
436 opts.Append(Form("workers=%s", fOptions.Get("workers").Data()));
438 Info("ProofHelper::PreSetup", "Connecting to %s with %soptions %s",
440 opts.IsNull() ? "no " : "",
442 TString proto(connect.GetProtocol());
443 if (proto.BeginsWith("lite") && fOptions.Has("workers"))
446 TProof::Open(connect.GetUrl(), opts);
447 // TProof::Open(connect.GetHost(), opts);
449 Error("ProofHelper::PreSetup", "Failed to open Proof connection %s",
454 // --- Check if we need to clear packages ------------------------
455 if (fOptions.Has("clear")) {
456 TString pkgs = fOptions.Get("clear");
457 if (pkgs.IsNull() || pkgs.EqualTo("all", TString::kIgnoreCase)) {
458 // No value given, clear all
459 if (gProof->ClearPackages() != 0)
460 Warning("ProofHelper::PreSetup", "Failed to lear all packages");
463 // Tokenize on ',' and clear each package
464 TObjArray* pars = pkgs.Tokenize(",");
467 while ((pkg = next())) {
468 if (gProof->ClearPackage(pkg->GetName()) != 0)
469 Warning("ProofHelper::PreSetup", "Failed to clear package %s",
478 * Set-up done after the task set-ups
480 * @return true on success
482 virtual Bool_t PostSetup()
484 AliAnalysisManager* mgr = AliAnalysisManager::GetAnalysisManager();
486 Error("ProofHelper::PostSetup", "No analysis manager defined");
490 // --- Check for output ------------------------------------------
491 if (fOptions.Has("dsname"))
492 OutputUtilities::RegisterDataset(fOptions.Get("dsname"));
493 if (fOptions.Has("storage"))
494 OutputUtilities::RegisterStorage(fOptions.Get("storage"));
496 // --- If we are not using PARs for Base, enable special PAR -----
498 TString tmp(fExtraLibs.Strip(TString::kBoth,':'));
499 TList* params = new TList;
500 params->SetOwner(true);
501 params->Add(new TNamed("ALIROOT_EXTRA_LIBS", tmp.Data()));
502 if (fOptions.Has("mode"))
503 params->Add(new TNamed("ALIROOT_MODE", fOptions.Get("mode").Data()));
505 params->Add(new TNamed("ALIROOT_MODE", "default"));
506 Int_t ret = gProof->EnablePackage(AliROOTParName(), params, true);
508 Error("ProofHelper::EnableAliROOT", "Failed to enable AliROOT PAR %s",
514 // --- Load par files --------------------------------------------
515 TString tmp = fExtraPars.Strip(TString::kBoth,':');
516 TObjArray* pars = tmp.Tokenize(":");
519 while ((obj = next())) {
520 // Enable the package, but do not build on client - already done
521 Int_t ret = gProof->EnablePackage(obj->GetName(), true);
523 Error("ProofHelper::PostSetup", "Failed to enable PAR %s",
529 // --- Load extra sources ----------------------------------------
530 TString tmp2 = fExtraSrcs.Strip(TString::kBoth, ':');
531 TObjArray* srcs = tmp2.Tokenize(":");
533 while ((obj = next())) {
534 Int_t ret = gProof->Load(Form("%s++g", obj->GetName()), true);
536 Error("ProofHelper::PostSetup", "Failed to compile %s", obj->GetName());
545 * @param nEvents Number of events to analyse
547 * @return The return value of AliAnalysisManager::StartAnalysis
549 virtual Long64_t Run(Long64_t nEvents=-1)
551 AliAnalysisManager* mgr = AliAnalysisManager::GetAnalysisManager();
552 gProof->SetLogLevel(TMath::Max(fVerbose-2,0),
553 /* TProofDebug::kPacketizer| */
555 /* TProofDebug::kSelector|
556 TProofDebug::kOutput|
558 TProofDebug::kGlobal|*/
559 TProofDebug::kPackage);
560 TString dsName(fUrl.GetFile());
561 // if (fUrl.GetAnchor() && fUrl.GetAnchor()[0] != '\0')
562 // dsName.Append(Form("#%s", fUrl.GetAnchor()));
563 Long64_t ret = mgr->StartAnalysis(fUrl.GetProtocol(), dsName, nEvents);
566 TProof::Mgr(fUrl.GetUrl())->GetSessionLogs()->Save("*","proof.log");
570 * Print information to standard output
574 virtual void Print(Option_t* option="") const
576 Helper::Print(option);
577 std::cout << std::boolalpha
578 << " --- Other settings -------\n"
579 << " Extra libraries : " << fExtraLibs << "\n"
580 << " Extra PARs : " << fExtraPars << "\n"
581 << " Extra sources : " << fExtraSrcs << "\n"
582 << " Use PARs of tasks: " << fUsePars << "\n"
583 << " Use PARs of base : " << fBasePars
584 << std::noboolalpha << std::endl;
589 * @return Path to output - possibly a data set
591 virtual TString OutputPath() const
594 if (fOptions.Has("dsname")) {
595 ret = Form("/%s/%s/", gProof->GetGroup(), gProof->GetUser());
596 ret.Append(OutputUtilities::RegisteredDataset());
601 * @return URL help string
603 virtual const Char_t* UrlHelp() const
605 return "proof://<host>[:<port>]/[<dataset>|<path>][?<options>][#<treeName>]";
608 * @return Short description
610 virtual const char* Desc() const { return "PROOF"; }