3 * @author Christian Holm Christensen <cholm@master.hehi.nbi.dk>
4 * @date Tue Oct 16 17:51:10 2012
6 * @brief PAR file utilities
8 * @ingroup pwglf_forward_trains_util
29 // ===================================================================
31 * Helper to set-up and load PARs
33 * @ingroup pwglf_forward_trains_util
38 * Find PAR file (either in current or parent directory or directly
39 * in $ALICE_ROOT), and link it here
41 * @param what PAR file name (sans .par)
43 * @return true on success
45 static Bool_t Find(const TString& what)
47 if (what.IsNull()) return false;
49 TString parFile(what);
50 if (!parFile.EndsWith(".par")) parFile.Append(".par");
51 if (gSystem->AccessPathName(parFile.Data())) {
54 if (gSystem->AccessPathName(Form("../%s", parFile.Data())) == 0)
55 src.Form("../%s", parFile.Data());
59 gSystem->ExpandPathName(Form("$(ALICE_ROOT)/%s", parFile.Data()));
60 if (gSystem->AccessPathName(aliParFile.Data()) == 0)
64 Error("ParUtilities::Find",
65 "PAR file %s not found in current or parent "
66 "directory nor in $(ALICE_ROOT)", parFile.Data());
69 // Copy to current directory
70 // TFile::Copy(aliParFile, parFile);
71 Info("", "Found PAR %s at %s", what.Data(), src.Data());
72 if (gSystem->Exec(Form("ln -s %s %s", src.Data(), parFile.Data())) != 0){
73 Error("ParUtilities::Find", "Failed to symlink %s to %s",
74 src.Data(), parFile.Data());
81 * Unpack and load a PAR file previously found with Find.
83 * @param name PAR file name
84 * @deprecated Use Find and Build instead
85 * @return true on success
87 static Bool_t Load(const TString& name)
89 if (name.IsNull()) return true;
91 Error("ParUtilities::Load", "No connection to a Proof cluster");
97 Info("ParUtilities::LoadLibrary", "Uploading %s", name.Data());
99 // First check in current directory
100 Int_t ret = gProof->UploadPackage(fn, TProof::kRemoveOld);
103 // IF not found there, then check parent directory
105 gSystem->ExpandPathName(fn);
106 ret = gProof->UploadPackage(fn);
110 // If not found in current or parent directory, try the
111 // the ALICE_ROOT directory
112 fn = Form("$ALICE_ROOT/%s.par", name.Data());
113 gSystem->ExpandPathName(fn);
114 ret = gProof->UploadPackage(fn);
118 // IF not found, bark
119 Error("ParUtilities::Load",
120 "Could not find module %s.par in current or parent directory "
121 "nor in $ALICE_ROOT", name.Data());
125 ret = gProof->EnablePackage(name);
126 Info("ParUtilities::Load", "Enabled package %s (from %s)",
127 name.Data(), fn.Data());
132 * Unpack, build, and load a PAR file.
134 * @param what Which PAR file
138 static Bool_t Build(const TString& what)
140 if (what.IsNull()) return false;
142 TString parFile(what);
143 if (!parFile.EndsWith(".par")) parFile.Append(".par");
146 gSystem->Exec(Form("tar xzf %s", parFile.Data()));
148 // Change directory into par archive
149 TString cwd = gSystem->WorkingDirectory();
152 if (dir.EndsWith(".par")) dir.ReplaceAll(".par", "");
153 if (!gSystem->ChangeDirectory(dir)) {
154 Error("ParUtilities::Setup", "Failed to change directory to %s",
160 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
161 Info("ParUtilities::Setup", "Building in PAR archive %s", parFile.Data());
162 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
163 Error("ParUtilities::Setup", "Failed to build in PAR directory %s",
165 gSystem->ChangeDirectory(cwd.Data());
170 // We need to make sure the current directory is in the load path
171 gSystem->SetDynamicPath(Form("%s:%s", gSystem->WorkingDirectory(),
172 gSystem->GetDynamicPath()));
173 // Check for setup script
174 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
175 // Info("ParUtilities::SetupPAR", "Setting up for PAR %s", what);
176 gROOT->Macro("PROOF-INF/SETUP.C");
178 if (!gSystem->ChangeDirectory(cwd.Data())) return false;
182 //__________________________________________________________________
185 * @name PAR generation from script
188 * Service function to make a PAR out of a script.
190 * The script should contain can contain a sub-class of AliAnalysisTask.
191 * The script will be compiled on the slaves before loading the
192 * AliAnalysisManager. Parts to (not) be compiled can be protected like
196 * // This will _only_ be compiled in the servers
199 * // This will not be compiled in the servers
203 * @param script Script to upload and compile in the PAR
204 * @param deps Dependency pars
205 * @param isLocal Local build
206 * @param helper Helper
208 * @return true on success.
210 static Bool_t MakeScriptPAR(Bool_t isLocal,
211 const TString& script,
215 TObjArray* depList = deps.Tokenize(", ");
217 // --- In local mode, just AcLic and load ------------------------
222 while ((dep = next()))
223 helper->LoadLibrary(dep->GetName());
226 Info("ParUtilities::MakeScriptPAR", "Loading macro %s", script.Data());
227 if (gROOT->LoadMacro(Form("%s++g", script.Data())) < 0) {
228 Error("ParUtilities::MakeScriptPAR",
229 "Failed to build local library %s", script.Data());
235 // --- Get the base name -----------------------------------------
236 Info("ParUtilities::MakeScriptPAR", "Making par file for %s",
238 TString base(gSystem->BaseName(script));
239 Int_t idx = base.Last('.');
240 if (idx != kNPOS) base.Remove(idx);
242 // --- Check name of script file ---------------------------------
245 if (script.EndsWith(".C")) ext = "C";
246 else if (script.EndsWith(".cxx")) ext = "cxx";
247 else { ext = "C"; scr.Append(".C"); }
249 // --- Check if we can access the file ---------------------------
250 TString path = TString::Format(".:%s", TROOT::GetMacroPath());
251 char* loc = gSystem->Which(path, scr);
253 Error("ParUtilities::MakeScriptPAR",
254 "Script %s not found in %s", scr.Data(), path.Data());
259 // --- Create our temporary directory ----------------------------
260 TString tmpdir(gSystem->TempDirectory());
261 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
262 char* templ = new char[ltempl];
263 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
264 if (!mkdtemp(templ)) {
265 Error("ParUtilities::MakeScriptPAR",
266 "Failed to generate temporary directory from template %s",
270 Info("", "Building PAR in %s", templ);
272 Bool_t retVal = false;
274 // --- Make directories for package ------------------------------
275 TString dir = TString::Format("%s/%s", templ, base.Data());
276 // Set-up directories
277 if (gSystem->MakeDirectory(dir) < 0)
278 throw TString::Format("Could not make directory '%s'", base.Data());
279 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
280 throw TString::Format("Could not make directory %s/PROOF-INF",
282 Info("", "Made directory %s", dir.Data());
284 // --- Copy the script to the setup directory --------------------
285 TString dest = TString::Format("%s/%s.%s", dir.Data(),
286 base.Data(), ext.Data());
287 Int_t ret = gSystem->CopyFile(full, dest, true);
289 case -1: throw TString::Format("Couldn't open %s for copy", scr.Data());
290 case -2: throw TString::Format("File %s exists", dest.Data());
291 case -3: throw TString::Format("Error while copying %s", scr.Data());
294 // --- Make scripts, etc. ----------------------------------------
295 if (!MakeScriptBuildScript(dir, base))
296 throw TString::Format("Failed to make build script");
297 if (!MakeScriptUtilityScript(dir))
298 throw TString::Format("Failed to make utility script");
299 if (!MakeScriptBuildMacro(dir, base, ext, depList))
300 throw TString::Format("Failed to make build macro");
301 if (!MakeScriptSetupMacro(dir, base, ext, depList))
302 throw TString::Format("Failed to setup macro");
304 // --- Pack up the archive ---------------------------------------
305 ret = gSystem->Exec(Form("(cd %s && tar -czf %s.par %s)",
306 templ, base.Data(),base.Data()));
308 throw TString::Format("Failed to create PAR file %s.PAR from %s",
309 base.Data(), dir.Data());
311 Info("", "Made par archive %s/%s.par - moving here",
313 // --- Move PAR file to here -------------------------------------
314 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, base.Data(),
317 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
318 templ, base.Data(), base.Data(),
319 gSystem->GetError());
323 Error("ParUtilities::MakeScriptPAR", "%s", e.Data());
327 // --- Remove temporary directory --------------------------------
328 gSystem->Exec(Form("rm -rf %s", templ));
333 * Write a build script
335 * @param dir Directory to put it in
336 * @param base Base name
338 * @return true on success
340 static Bool_t MakeScriptBuildScript(const TString& dir,
343 // Make our build file
344 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
346 Error("ParUtilities::MakeScriptBuildScript",
347 "Failed to open out shell script");
351 << "if test x$ALICE_ROOT != x ; then\n"
352 << " if test x$ALICE_TARGET = x ; then\n"
353 << " export ALICE_TARGET=`$ROOTSYS/bin/root-config --arch`\n"
355 << " export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:"
356 << "${ALICE_ROOT}/lib/tgt_${ALICE_TARGET}\n"
358 << "echo BUILD.sh@`hostname`: Building " << base << "\n"
359 << "root.exe -l -out -q PROOF-INF/BUILD.C 2>&1 | tee "
361 << "echo BUILD.sh@`hostname`: done: $?\n"
364 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755) != 0) {
365 Error("ParUtilities::MakeScriptBuildScript",
366 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
373 * Write a build macro
375 * @param dir Directory to put macro in
376 * @param deps Dependencies
377 * @param base Base name of script to compile
378 * @param ext `extension' - last part of file name
380 * @return true on success
382 static Bool_t MakeScriptBuildMacro(const TString& dir,
385 const TCollection* deps) {
386 std::ofstream out(Form("%s/PROOF-INF/BUILD.C", dir.Data()));
388 Error("ParUtilities::MakeScriptBuildMacro","Failed to open build script");
391 out << "void BUILD() {\n"
392 << " gSystem->AddIncludePath(\"-DBUILD_PAR=1\");\n"
393 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
394 << " LoadROOTLibs();\n"
395 << " AddAliROOT();\n";
399 while ((dep = next())) {
400 out << " AddDep(\"" << dep->GetName() << "\");\t"
401 << " LoadDep(\"" << dep->GetName() << "\");\n";
404 out << " // gDebug = 5;\n"
405 << " int ret = gROOT->LoadMacro(\""
406 << base << "." << ext << "++g\");\n"
407 << " if (ret != 0) Fatal(\"BUILD\",\"Failed to build\");\n"
408 << " // else Info(\"BUILD\", \"Made " << base << "\");\n"
416 * Make a utility macro
418 * @param dir Directory to put the macro in
420 * @return true on success
422 static Bool_t MakeScriptUtilityScript(const TString& dir)
424 std::ofstream out(Form("%s/PROOF-INF/UTIL.C", dir.Data()));
426 Error("ParUtilities::MakeScriptUtilityScript",
427 "Failed to open utility script");
430 out << "void LoadROOTLibs() {\n"
431 << " gSystem->Load(\"libVMC\");\n"
432 << " gSystem->Load(\"libNet\");\n"
433 << " gSystem->Load(\"libTree\");\n"
434 << " gSystem->Load(\"libPhysics\");\n"
435 << " gSystem->Load(\"libMinuit\");\n"
437 << "void AddAliROOT() {\n"
438 << " TString val(gSystem->Getenv(\"ALICE_ROOT\"));\n"
439 << " if (val.IsNull())\n"
440 << " Warning(\"Add\",\"ALICE_ROOT not defined\");\n"
442 << " gSystem->AddIncludePath(Form(\"-I%s/include\",val.Data()));\n"
444 << "void AddDep(const char* env) {\n"
445 << " TString val(gSystem->Getenv(Form(\"%s_INCLUDE\",env)));\n"
446 << " if (val.IsNull())\n"
447 << " Warning(\"Add\",\"%s_INCLUDE not defined\", env);\n"
449 << " gSystem->AddIncludePath(Form(\"-I../%s\",val.Data()));\n"
452 << "void LoadDep(const char* name) {\n"
453 << " gSystem->AddDynamicPath(Form(\"../%s\",name));\n"
454 << " char* full = gSystem->DynamicPathName(name,true);\n"
456 << " full = gSystem->DynamicPathName(Form(\"lib%s\",name),true);\n"
458 << " full = gSystem->DynamicPathName(Form(\"lib%s.so\",name),true);\n"
460 << " Warning(\"LoadDep\",\"Module %s not found\", name);\n"
463 << " gSystem->Load(full);\n"
470 * Make a setup script
472 * @param dir Directory to put it in
473 * @param base Base name of target script
474 * @param ext Extension of target script
475 * @param deps Dependencies
477 * @return true on success
479 static Bool_t MakeScriptSetupMacro(const TString& dir,
482 const TCollection* deps)
484 // Make our set-up script
485 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
487 Error("ParUtilities::MakeScriptSetupMacro",
488 "Failed to open setup script");
491 out << "void SETUP() {\n"
492 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
493 << " LoadROOTLibs();\n"
494 << " // Info(\"SETUP\",\"Loading libraries\");\n";
498 while ((dep = next()))
499 out << " LoadDep(\"" << dep->GetName() << "\");\n";
501 out << " // gDebug = 5;\n"
502 << " // Info(\"SETUP\",\"Loading " << base <<"_"<< ext << ".so\");\n"
503 << " gSystem->Load(\"" << base << "_" << ext << ".so\");\n"
504 << " // gDebug = 0;\n"
505 << " gROOT->ProcessLine(\".include " << base << "\");\n"
506 << " gSystem->Setenv(\"" << base << "_INCLUDE\",\""
508 << " // Info(\"SETUP\", \"Done\");\n"
515 //__________________________________________________________________
518 * @name PAR generation from aux file list
520 static Bool_t MakeAuxFilePAR(const TList& files,
522 Bool_t verbose=false)
524 // --- Check input -----------------------------------------------
525 if (files.GetEntries() <= 0) return true;
527 // --- Create our temporary directory ----------------------------
528 Bool_t retval = true;
529 TString tmpdir(gSystem->TempDirectory());
530 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
531 char* templ = new char[ltempl];
532 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
533 if (!mkdtemp(templ)) {
534 Error("ParUtilities::MakeAuxFilePAR",
535 "Failed to generate temporary directory from template %s",
539 if (verbose) Printf("Preparing PAR file in %s", templ);
542 // --- Make directories for package ------------------------------
543 TString dir = TString::Format("%s/%s", templ, name.Data());
544 // Set-up directories
545 if (gSystem->MakeDirectory(dir) < 0)
546 throw TString::Format("Could not make directory '%s'", name.Data());
547 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
548 throw TString::Format("Could not make directory %s/PROOF-INF",
553 while ((o = next())) {
554 TString fn(o->GetName());
555 if (verbose) Printf("Got %s", fn.Data());
556 if (fn.BeginsWith("/")) {
557 Warning("MakeAuxFilePAR", "Will not include absolute path %s",
559 continue; // absolute path
562 if (gSystem->AccessPathName(fn.Data())) {
563 Warning("MakeAuxFilePAR", "Cannot access %s", fn.Data());
564 continue; // non-exist
566 // Loop over path components and make directories as needed
567 TObjArray* comps = fn.Tokenize("/");
569 Int_t n = comps->GetEntriesFast();
570 if (verbose) Printf("Got %d path components in %s", n-1, fn.Data());
572 for (Int_t i = 0; i < n-1; i++) {
573 TObjString* comp = static_cast<TObjString*>(comps->At(i));
574 TString& c = comp->String();
575 if (c.IsNull()) continue;
576 if (c.EqualTo(".")) continue;
578 Bool_t doMake = true;
579 if (c.EqualTo("..")) { doMake = false; lvl--; }
581 cur = gSystem->ConcatFileName(cur, c);
583 Warning("MakeAuxFilePAR", "Path %s points outside archive, ignored",
590 if (!gSystem->AccessPathName(cur)) continue;
591 if (verbose) Printf("Making directory %s", cur.Data());
592 gSystem->MakeDirectory(cur);
595 if (verbose) Printf("cur=%s for %s lvl=%d", cur.Data(), fn.Data(), lvl);
597 if (lvl < 0) continue;
599 TString dest = TString::Format("%s/%s", cur.Data(),
600 gSystem->BaseName(fn.Data()));
601 if (verbose) Printf("%s -> %s", fn.Data(), dest.Data());
602 Int_t ret = gSystem->CopyFile(fn, dest, true);
604 case -1: throw TString::Format("Couldn't open %s for copy", fn.Data());
605 case -2: throw TString::Format("File %s exists", dest.Data());
606 case -3: throw TString::Format("Error while copying %s", fn.Data());
611 // Make our build file
612 if (verbose) Printf("Making build script");
613 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
615 Error("ParUtilities::MakeAuxFilePAR",
616 "Failed to open out shell script");
619 out << "#!/bin/sh\n\n"
620 << "echo \"Nothing to be done\"\n\n"
621 << "# EOF" << std::endl;
623 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755)) {
624 Error("ParUtilities::MakeAuxFilePAR",
625 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
631 if (verbose) Printf("Making setup script");
632 // Make our setup file
633 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
635 Error("ParUtilities::MakeAuxFilePAR",
636 "Failed to open out ROOT script");
639 // The SETUP script is executed in the package's directory in
640 // the package cache - not in the session directory. Hence,
641 // we take special care to get a link to the session directory
642 // from the package cache directory
643 out << "void SETUP()\n"
645 << " TString oldDir(gSystem->WorkingDirectory());\n"
646 << " TSystemDirectory* dir = new TSystemDirectory(\".\",\".\");\n"
647 << " TList* files = dir->GetListOfFiles();\n"
648 << " if (!gSystem->ChangeDirectory(oldDir)) {\n"
649 << " Error(\"SETUP\", \"Failed to go back to %s\",\n"
650 << " oldDir.Data());\n"
653 << " if (!files) {\n"
654 << " Warning(\"SETUP\", \"No files\");\n"
655 << " gSystem->Exec(\"pwd; ls -al\");\n"
658 << " files->Sort();\n"
659 << " TString pkgDir = gSystem->WorkingDirectory();\n"
660 << " TString sesDir = gProofServ->GetSessionDir();\n"
661 << " Info(\"\",\"Session dir: %s\",sesDir);\n"
662 << " TIter next(files);\n"
663 << " TSystemFile* file = 0;\n"
664 << " while ((file = static_cast<TSystemFile*>(next()))) {\n"
665 << " TString name(file->GetName());\n"
666 << " if (name == \".\" || name == \"..\") continue;\n"
667 << " TString title(file->GetTitle());\n"
668 << " TString full(gSystem->ConcatFileName(pkgDir.Data(),\n"
669 << " name.Data()));\n"
670 << " TString tgt(gSystem->ConcatFileName(sesDir.Data(),\n"
671 << " name.Data()));\n"
672 << " if (file->IsA()->InheritsFrom(TSystemDirectory::Class())){\n"
673 << " gSystem->mkdir(tgt.Data(), true);\n"
676 << " Info(\"\",\"Linking %s to %s\",full.Data(),tgt.Data());\n"
677 << " gSystem->Symlink(full, tgt);\n"
680 << "// EOF " << std::endl;
683 if (verbose) Printf("Packing up");
685 ret = gSystem->Exec(Form("(cd %s && tar -c%szf %s.par %s)",
686 templ, (verbose ? "v" : ""),
687 name.Data(),name.Data()));
689 throw TString::Format("Failed to create PAR file %s.PAR from %s",
690 name.Data(), name.Data());
692 // --- Move PAR file to here -------------------------------------
693 if (verbose) Printf("Move here");
694 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, name.Data(),
697 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
698 templ, name.Data(), name.Data(),
699 gSystem->GetError());
703 Printf("List content");
704 gSystem->Exec(Form("tar tzf %s.par", name.Data()));
709 Error("ParUtilities::MakeAuxFilePAR", "%s", e.Data());
713 // --- Remove temporary directory --------------------------------
714 gSystem->Exec(Form("rm -rf %s", templ));