3 * @author Christian Holm Christensen <cholm@master.hehi.nbi.dk>
4 * @date Tue Oct 16 17:51:10 2012
6 * @brief PAR file utilities
8 * @ingroup pwglf_forward_trains_util
29 // ===================================================================
31 * Helper to set-up and load PARs
33 * @ingroup pwglf_forward_trains_util
38 * Find PAR file (either in current or parent directory or directly
39 * in $ALICE_ROOT), and link it here
41 * @param what PAR file name (sans .par)
43 * @return true on success
45 static Bool_t Find(const TString& what)
47 if (what.IsNull()) return false;
49 TString parFile(what);
50 if (!parFile.EndsWith(".par")) parFile.Append(".par");
51 if (gSystem->AccessPathName(parFile.Data())) {
54 if (gSystem->AccessPathName(Form("../%s", parFile.Data())) == 0)
55 src.Form("../%s", parFile.Data());
59 gSystem->ExpandPathName(Form("$(ALICE_ROOT)/%s", parFile.Data()));
60 if (gSystem->AccessPathName(aliParFile.Data()) == 0)
64 Error("ParUtilities::Find",
65 "PAR file %s not found in current or parent "
66 "directory nor in $(ALICE_ROOT)", parFile.Data());
69 // Copy to current directory
70 // TFile::Copy(aliParFile, parFile);
71 Info("", "Found PAR %s at %s", what.Data(), src.Data());
72 if (gSystem->Exec(Form("ln -s %s %s", src.Data(), parFile.Data())) != 0){
73 Error("ParUtilities::Find", "Failed to symlink %s to %s",
74 src.Data(), parFile.Data());
81 * Unpack and load a PAR file previously found with Find.
83 * @param name PAR file name
84 * @deprecated Use Find and Build instead
85 * @return true on success
87 static Bool_t Load(const TString& name)
89 if (name.IsNull()) return true;
91 Error("ParUtilities::Load", "No connection to a Proof cluster");
97 Info("ParUtilities::LoadLibrary", "Uploading %s", name.Data());
99 // First check in current directory
100 Int_t ret = gProof->UploadPackage(fn, TProof::kRemoveOld);
103 // IF not found there, then check parent directory
105 gSystem->ExpandPathName(fn);
106 ret = gProof->UploadPackage(fn);
110 // If not found in current or parent directory, try the
111 // the ALICE_ROOT directory
112 fn = Form("$ALICE_ROOT/%s.par", name.Data());
113 gSystem->ExpandPathName(fn);
114 ret = gProof->UploadPackage(fn);
118 // IF not found, bark
119 Error("ParUtilities::Load",
120 "Could not find module %s.par in current or parent directory "
121 "nor in $ALICE_ROOT", name.Data());
125 ret = gProof->EnablePackage(name);
126 Info("ParUtilities::Load", "Enabled package %s (from %s)",
127 name.Data(), fn.Data());
132 * Unpack, build, and load a PAR file.
134 * @param what Which PAR file
138 static Bool_t Build(const TString& what)
140 if (what.IsNull()) return false;
142 TString parFile(what);
143 if (!parFile.EndsWith(".par")) parFile.Append(".par");
146 gSystem->Exec(Form("tar xzf %s", parFile.Data()));
148 // Change directory into par archive
149 TString cwd = gSystem->WorkingDirectory();
152 if (dir.EndsWith(".par")) dir.ReplaceAll(".par", "");
153 if (!gSystem->ChangeDirectory(dir)) {
154 Error("ParUtilities::Setup", "Failed to change directory to %s",
160 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
161 Info("ParUtilities::Setup", "Building in PAR archive %s", parFile.Data());
162 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
163 Error("ParUtilities::Setup", "Failed to build in PAR directory %s",
165 gSystem->ChangeDirectory(cwd.Data());
170 // We need to make sure the current directory is in the load path
171 gSystem->SetDynamicPath(Form("%s:%s", gSystem->WorkingDirectory(),
172 gSystem->GetDynamicPath()));
173 // Check for setup script
174 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
175 // Info("ParUtilities::SetupPAR", "Setting up for PAR %s", what);
176 gROOT->Macro("PROOF-INF/SETUP.C");
178 if (!gSystem->ChangeDirectory(cwd.Data())) return false;
182 //__________________________________________________________________
185 * @name PAR generation from script
188 * Service function to make a PAR out of a script.
190 * The script should contain can contain a sub-class of AliAnalysisTask.
191 * The script will be compiled on the slaves before loading the
192 * AliAnalysisManager. Parts to (not) be compiled can be protected like
196 * // This will _only_ be compiled in the servers
199 * // This will not be compiled in the servers
203 * @param script Script to upload and compile in the PAR
204 * @param deps Dependency pars
205 * @param isLocal Local build
206 * @param helper Helper
208 * @return true on success.
210 static Bool_t MakeScriptPAR(Bool_t isLocal,
211 const TString& script,
215 TObjArray* depList = deps.Tokenize(", ");
217 // --- In local mode, just AcLic and load ------------------------
222 while ((dep = next()))
223 helper->LoadLibrary(dep->GetName());
226 Info("ParUtilities::MakeScriptPAR", "Loading macro %s", script.Data());
227 if (gROOT->LoadMacro(Form("%s++g", script.Data())) < 0) {
228 Error("ParUtilities::MakeScriptPAR",
229 "Failed to build local library %s", script.Data());
235 // --- Get the base name -----------------------------------------
236 Info("ParUtilities::MakeScriptPAR", "Making par file for %s",
238 TString base(gSystem->BaseName(script));
239 Int_t idx = base.Last('.');
240 if (idx != kNPOS) base.Remove(idx);
242 // --- Check name of script file ---------------------------------
245 if (script.EndsWith(".C")) ext = "C";
246 else if (script.EndsWith(".cxx")) ext = "cxx";
247 else { ext = "C"; scr.Append(".C"); }
249 // --- Check if we can access the file ---------------------------
250 TString path = TString::Format(".:%s", TROOT::GetMacroPath());
251 char* loc = gSystem->Which(path, scr);
253 Error("ParUtilities::MakeScriptPAR",
254 "Script %s not found in %s", scr.Data(), path.Data());
259 // --- Create our temporary directory ----------------------------
260 TString tmpdir(gSystem->TempDirectory());
261 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
262 char* templ = new char[ltempl];
263 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
264 if (!mkdtemp(templ)) {
265 Error("ParUtilities::MakeScriptPAR",
266 "Failed to generate temporary directory from template %s",
270 Info("", "Building PAR in %s", templ);
272 Bool_t retVal = false;
274 // --- Make directories for package ------------------------------
275 TString dir = TString::Format("%s/%s", templ, base.Data());
276 // Set-up directories
277 if (gSystem->MakeDirectory(dir) < 0)
278 throw TString::Format("Could not make directory '%s'", base.Data());
279 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
280 throw TString::Format("Could not make directory %s/PROOF-INF",
282 Info("", "Made directory %s", dir.Data());
284 // --- Copy the script to the setup directory --------------------
285 TString dest = TString::Format("%s/%s.%s", dir.Data(),
286 base.Data(), ext.Data());
287 Int_t ret = gSystem->CopyFile(full, dest, true);
289 case -1: throw TString::Format("Couldn't open %s for copy", scr.Data());
290 case -2: throw TString::Format("File %s exists", dest.Data());
291 case -3: throw TString::Format("Error while copying %s", scr.Data());
294 // --- Make scripts, etc. ----------------------------------------
295 if (!MakeScriptBuildScript(dir, base))
296 throw TString::Format("Failed to make build script");
297 if (!MakeScriptUtilityScript(dir))
298 throw TString::Format("Failed to make utility script");
299 if (!MakeScriptBuildMacro(dir, base, ext, depList))
300 throw TString::Format("Failed to make build macro");
301 if (!MakeScriptSetupMacro(dir, base, ext, depList))
302 throw TString::Format("Failed to setup macro");
304 // --- Pack up the archive ---------------------------------------
305 ret = gSystem->Exec(Form("(cd %s && tar -czf %s.par %s)",
306 templ, base.Data(),base.Data()));
308 throw TString::Format("Failed to create PAR file %s.PAR from %s",
309 base.Data(), dir.Data());
311 Info("", "Made par archive %s/%s.par - moving here",
313 // --- Move PAR file to here -------------------------------------
314 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, base.Data(),
317 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
318 templ, base.Data(), base.Data(),
319 gSystem->GetError());
323 Error("ParUtilities::MakeScriptPAR", "%s", e.Data());
327 // --- Remove temporary directory --------------------------------
328 gSystem->Exec(Form("rm -rf %s", templ));
333 * Write a build script
335 * @param dir Directory to put it in
336 * @param base Base name
338 * @return true on success
340 static Bool_t MakeScriptBuildScript(const TString& dir,
343 // Make our build file
344 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
346 Error("ParUtilities::MakeScriptBuildScript",
347 "Failed to open out shell script");
351 << "if test x$ALICE_ROOT != x ; then\n"
352 << " if test x$ALICE_TARGET = x ; then\n"
353 << " export ALICE_TARGET=`$ROOTSYS/bin/root-config --arch`\n"
355 << " export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:"
356 << "${ALICE_ROOT}/lib/tgt_${ALICE_TARGET}\n"
358 << "# printenv | sort -u\n"
359 << "echo BUILD.sh@`hostname`: Building " << base << "\n"
360 << "root.exe -l -out -q PROOF-INF/BUILD.C 2>&1 | tee "
362 << "echo BUILD.sh@`hostname`: done: $?\n"
365 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755) != 0) {
366 Error("ParUtilities::MakeScriptBuildScript",
367 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
374 * Write a build macro
376 * @param dir Directory to put macro in
377 * @param deps Dependencies
378 * @param base Base name of script to compile
379 * @param ext `extension' - last part of file name
381 * @return true on success
383 static Bool_t MakeScriptBuildMacro(const TString& dir,
386 const TCollection* deps) {
387 std::ofstream out(Form("%s/PROOF-INF/BUILD.C", dir.Data()));
389 Error("ParUtilities::MakeScriptBuildMacro","Failed to open build script");
392 out << "void BUILD() {\n"
393 << " gSystem->AddIncludePath(\"-DBUILD_PAR=1\");\n"
394 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
395 << " LoadROOTLibs();\n"
396 << " AddAliROOT();\n";
400 while ((dep = next())) {
401 out << " AddDep(\"" << dep->GetName() << "\");\t"
402 << " LoadDep(\"" << dep->GetName() << "\");\n";
405 out << " // gDebug = 5;\n"
406 << " // gSystem->ListLibraries();\n"
407 << " int ret = gROOT->LoadMacro(\""
408 << base << "." << ext << "++g\");\n"
409 << " if (ret != 0) Fatal(\"BUILD\",\"Failed to build\");\n"
410 << " // else Info(\"BUILD\", \"Made " << base << "\");\n"
418 * Make a utility macro
420 * @param dir Directory to put the macro in
422 * @return true on success
424 static Bool_t MakeScriptUtilityScript(const TString& dir)
426 std::ofstream out(Form("%s/PROOF-INF/UTIL.C", dir.Data()));
428 Error("ParUtilities::MakeScriptUtilityScript",
429 "Failed to open utility script");
432 out << "void LoadROOTLibs() {\n"
433 << " gSystem->Load(\"libVMC\");\n"
434 << " gSystem->Load(\"libNet\");\n"
435 << " gSystem->Load(\"libTree\");\n"
436 << " gSystem->Load(\"libPhysics\");\n"
437 << " gSystem->Load(\"libMinuit\");\n"
439 << "void AddAliROOT() {\n"
440 << " TString val(gSystem->Getenv(\"ALICE_ROOT\"));\n"
441 << " if (val.IsNull())\n"
442 << " Warning(\"Add\",\"ALICE_ROOT not defined\");\n"
444 << " gSystem->AddIncludePath(Form(\"-I%s/include\",val.Data()));\n"
446 << "void AddDep(const char* env) {\n"
447 << " TString val(gSystem->Getenv(Form(\"%s_INCLUDE\",env)));\n"
448 << " if (val.IsNull())\n"
449 << " Warning(\"Add\",\"%s_INCLUDE not defined\", env);\n"
451 << " // gSystem->AddIncludePath(Form(\"-I../%s\",val.Data()));\n"
452 << " // Prepend to include path\n"
453 << " TString incPath(gSystem->GetIncludePath());\n"
454 << " incPath.Prepend(Form(\"-I../%s \",val.Data()));\n"
455 << " gSystem->SetIncludePath(incPath);\n"
456 << " // Printf(\"Include path: %s\",incPath.Data());\n"
459 << "void LoadDep(const char* name) {\n"
460 << " // Printf(\"Loading dependency \\\"%s\\\" ...\",name);\n"
461 << " // gSystem->AddDynamicPath(Form(\"../%s\",name));\n"
462 << " // Prepend to dynamic path\n"
463 << " TString dynPath(gSystem->GetDynamicPath());\n"
464 << " dynPath.Prepend(Form(\"../%s:\",name));\n"
465 << " gSystem->SetDynamicPath(dynPath);\n"
466 << " // Printf(\"Dynamic path: %s\",dynPath.Data());\n"
467 << " char* full = gSystem->DynamicPathName(name,true);\n"
469 << " full = gSystem->DynamicPathName(Form(\"lib%s\",name),true);\n"
471 << " full = gSystem->DynamicPathName(Form(\"lib%s.so\",name),true);\n"
473 << " full = gSystem->DynamicPathName(Form(\"%s_C.so\",name),true);\n"
475 << " full = gSystem->DynamicPathName(Form(\"%s_h.so\",name),true);\n"
477 << " Warning(\"LoadDep\",\"Module %s not found\", name);\n"
480 << " Printf(\"Loading \\\"%s\\\" for \\\"%s\\\"\",full,name);\n"
481 << " gSystem->Load(full);\n"
488 * Make a setup script
490 * @param dir Directory to put it in
491 * @param base Base name of target script
492 * @param ext Extension of target script
493 * @param deps Dependencies
495 * @return true on success
497 static Bool_t MakeScriptSetupMacro(const TString& dir,
500 const TCollection* deps)
502 // Make our set-up script
503 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
505 Error("ParUtilities::MakeScriptSetupMacro",
506 "Failed to open setup script");
509 out << "void SETUP() {\n"
510 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
511 << " Printf(\"Loading \\\"" << base << "\\\" ...\");\n"
512 << " LoadROOTLibs();\n"
513 << " // Info(\"SETUP\",\"Loading libraries\");\n";
517 while ((dep = next()))
518 out << " LoadDep(\"" << dep->GetName() << "\");\n";
520 out << " // gDebug = 5;\n"
521 << " // Info(\"SETUP\",\"Loading " << base <<"_"<< ext << ".so\");\n"
522 << " gSystem->Load(\"" << base << "_" << ext << ".so\");\n"
523 << " // gDebug = 0;\n"
524 << " gROOT->ProcessLine(\".include " << base << "\");\n"
525 << " gSystem->Setenv(\"" << base << "_INCLUDE\",\""
527 << " // Info(\"SETUP\", \"Done\");\n"
534 //__________________________________________________________________
537 * @name PAR generation from aux file list
539 static Bool_t MakeAuxFilePAR(const TList& files,
541 Bool_t verbose=false)
543 // --- Check input -----------------------------------------------
544 if (files.GetEntries() <= 0) return true;
546 // --- Create our temporary directory ----------------------------
547 Bool_t retval = true;
548 TString tmpdir(gSystem->TempDirectory());
549 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
550 char* templ = new char[ltempl];
551 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
552 if (!mkdtemp(templ)) {
553 Error("ParUtilities::MakeAuxFilePAR",
554 "Failed to generate temporary directory from template %s",
558 if (verbose) Printf("Preparing PAR file in %s", templ);
561 // --- Make directories for package ------------------------------
562 TString dir = TString::Format("%s/%s", templ, name.Data());
563 // Set-up directories
564 if (gSystem->MakeDirectory(dir) < 0)
565 throw TString::Format("Could not make directory '%s'", name.Data());
566 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
567 throw TString::Format("Could not make directory %s/PROOF-INF",
572 while ((o = next())) {
573 TString fn(o->GetName());
574 if (verbose) Printf("Got %s", fn.Data());
575 if (fn.BeginsWith("/")) {
576 Warning("MakeAuxFilePAR", "Will not include absolute path %s",
578 continue; // absolute path
581 if (gSystem->AccessPathName(fn.Data())) {
582 Warning("MakeAuxFilePAR", "Cannot access %s", fn.Data());
583 continue; // non-exist
585 // Loop over path components and make directories as needed
586 TObjArray* comps = fn.Tokenize("/");
588 Int_t n = comps->GetEntriesFast();
589 if (verbose) Printf("Got %d path components in %s", n-1, fn.Data());
591 for (Int_t i = 0; i < n-1; i++) {
592 TObjString* comp = static_cast<TObjString*>(comps->At(i));
593 TString& c = comp->String();
594 if (c.IsNull()) continue;
595 if (c.EqualTo(".")) continue;
597 Bool_t doMake = true;
598 if (c.EqualTo("..")) { doMake = false; lvl--; }
600 cur = gSystem->ConcatFileName(cur, c);
602 Warning("MakeAuxFilePAR", "Path %s points outside archive, ignored",
609 if (!gSystem->AccessPathName(cur)) continue;
610 if (verbose) Printf("Making directory %s", cur.Data());
611 gSystem->MakeDirectory(cur);
614 if (verbose) Printf("cur=%s for %s lvl=%d", cur.Data(), fn.Data(), lvl);
616 if (lvl < 0) continue;
618 TString dest = TString::Format("%s/%s", cur.Data(),
619 gSystem->BaseName(fn.Data()));
620 if (verbose) Printf("%s -> %s", fn.Data(), dest.Data());
621 Int_t ret = gSystem->CopyFile(fn, dest, true);
623 case -1: throw TString::Format("Couldn't open %s for copy", fn.Data());
624 case -2: throw TString::Format("File %s exists", dest.Data());
625 case -3: throw TString::Format("Error while copying %s", fn.Data());
630 // Make our build file
631 if (verbose) Printf("Making build script");
632 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
634 Error("ParUtilities::MakeAuxFilePAR",
635 "Failed to open out shell script");
638 out << "#!/bin/sh\n\n"
639 << "echo \"Nothing to be done\"\n\n"
640 << "# EOF" << std::endl;
642 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755)) {
643 Error("ParUtilities::MakeAuxFilePAR",
644 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
650 if (verbose) Printf("Making setup script");
651 // Make our setup file
652 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
654 Error("ParUtilities::MakeAuxFilePAR",
655 "Failed to open out ROOT script");
658 // The SETUP script is executed in the package's directory in
659 // the package cache - not in the session directory. Hence,
660 // we take special care to get a link to the session directory
661 // from the package cache directory
662 out << "void SETUP()\n"
664 << " TString oldDir(gSystem->WorkingDirectory());\n"
665 << " TSystemDirectory* dir = new TSystemDirectory(\".\",\".\");\n"
666 << " TList* files = dir->GetListOfFiles();\n"
667 << " if (!gSystem->ChangeDirectory(oldDir)) {\n"
668 << " Error(\"SETUP\", \"Failed to go back to %s\",\n"
669 << " oldDir.Data());\n"
672 << " if (!files) {\n"
673 << " Warning(\"SETUP\", \"No files\");\n"
674 << " gSystem->Exec(\"pwd; ls -al\");\n"
677 << " files->Sort();\n"
678 << " TString pkgDir = gSystem->WorkingDirectory();\n"
679 << " TString sesDir = gProofServ->GetSessionDir();\n"
680 << " Info(\"\",\"Session dir: %s\",sesDir);\n"
681 << " TIter next(files);\n"
682 << " TSystemFile* file = 0;\n"
683 << " while ((file = static_cast<TSystemFile*>(next()))) {\n"
684 << " TString name(file->GetName());\n"
685 << " if (name == \".\" || name == \"..\") continue;\n"
686 << " TString title(file->GetTitle());\n"
687 << " TString full(gSystem->ConcatFileName(pkgDir.Data(),\n"
688 << " name.Data()));\n"
689 << " TString tgt(gSystem->ConcatFileName(sesDir.Data(),\n"
690 << " name.Data()));\n"
691 << " if (file->IsA()->InheritsFrom(TSystemDirectory::Class())){\n"
692 << " gSystem->mkdir(tgt.Data(), true);\n"
695 << " Info(\"\",\"Linking %s to %s\",full.Data(),tgt.Data());\n"
696 << " gSystem->Symlink(full, tgt);\n"
699 << "// EOF " << std::endl;
702 if (verbose) Printf("Packing up");
704 ret = gSystem->Exec(Form("(cd %s && tar -c%szf %s.par %s)",
705 templ, (verbose ? "v" : ""),
706 name.Data(),name.Data()));
708 throw TString::Format("Failed to create PAR file %s.PAR from %s",
709 name.Data(), name.Data());
711 // --- Move PAR file to here -------------------------------------
712 if (verbose) Printf("Move here");
713 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, name.Data(),
716 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
717 templ, name.Data(), name.Data(),
718 gSystem->GetError());
722 Printf("List content");
723 gSystem->Exec(Form("tar tzf %s.par", name.Data()));
728 Error("ParUtilities::MakeAuxFilePAR", "%s", e.Data());
732 // --- Remove temporary directory --------------------------------
733 gSystem->Exec(Form("rm -rf %s", templ));