3 * @author Christian Holm Christensen <cholm@master.hehi.nbi.dk>
4 * @date Tue Oct 16 17:51:10 2012
6 * @brief PAR file utilities
8 * @ingroup pwglf_forward_trains_util
29 // ===================================================================
31 * Helper to set-up and load PARs
33 * @ingroup pwglf_forward_trains_util
38 * Find PAR file (either in current or parent directory or directly
39 * in $ALICE_ROOT), and link it here
41 * @param what PAR file name (sans .par)
43 * @return true on success
45 static Bool_t Find(const TString& what)
47 if (what.IsNull()) return false;
49 TString parFile(what);
50 if (!parFile.EndsWith(".par")) parFile.Append(".par");
51 if (gSystem->AccessPathName(parFile.Data())) {
54 if (gSystem->AccessPathName(Form("../%s.par", parFile.Data())) == 0)
55 src.Form("../%s", parFile.Data());
59 gSystem->ExpandPathName(Form("$(ALICE_ROOT)/%s", parFile.Data()));
60 if (gSystem->AccessPathName(aliParFile.Data()) == 0)
64 Error("ParUtilities::Find",
65 "PAR file %s not found in current or parent "
66 "directory nor in $(ALICE_ROOT)", parFile.Data());
69 // Copy to current directory
70 // TFile::Copy(aliParFile, parFile);
71 if (gSystem->Exec(Form("ln -s %s %s", src.Data(), parFile.Data())) != 0){
72 Error("ParUtilities::Find", "Failed to symlink %s to %s",
73 src.Data(), parFile.Data());
80 * Unpack and load a PAR file previously found with Find.
82 * @param name PAR file name
83 * @deprecated Use Find and Build instead
84 * @return true on success
86 static Bool_t Load(const TString& name)
88 if (name.IsNull()) return true;
90 Error("ParUtilities::Load", "No connection to a Proof cluster");
96 Info("ParUtilities::LoadLibrary", "Uploading %s", name.Data());
98 // First check in current directory
99 Int_t ret = gProof->UploadPackage(fn, TProof::kRemoveOld);
102 // IF not found there, then check parent directory
104 gSystem->ExpandPathName(fn);
105 ret = gProof->UploadPackage(fn);
109 // If not found in current or parent directory, try the
110 // the ALICE_ROOT directory
111 fn = Form("$ALICE_ROOT/%s.par", name.Data());
112 gSystem->ExpandPathName(fn);
113 ret = gProof->UploadPackage(fn);
117 // IF not found, bark
118 Error("ParUtilities::Load",
119 "Could not find module %s.par in current or parent directory "
120 "nor in $ALICE_ROOT", name.Data());
124 ret = gProof->EnablePackage(name);
125 Info("ParUtilities::Load", "Enabled package %s (from %s)",
126 name.Data(), fn.Data());
131 * Unpack, build, and load a PAR file.
133 * @param what Which PAR file
137 static Bool_t Build(const TString& what)
139 if (what.IsNull()) return false;
141 TString parFile(what);
142 if (!parFile.EndsWith(".par")) parFile.Append(".par");
145 gSystem->Exec(Form("tar xzf %s", parFile.Data()));
147 // Change directory into par archive
148 TString cwd = gSystem->WorkingDirectory();
151 if (dir.EndsWith(".par")) dir.ReplaceAll(".par", "");
152 if (!gSystem->ChangeDirectory(dir)) {
153 Error("ParUtilities::Setup", "Failed to change directory to %s",
159 if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
160 Info("ParUtilities::Setup", "Building in PAR archive %s", parFile.Data());
161 if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
162 Error("ParUtilities::Setup", "Failed to build in PAR directory %s",
164 gSystem->ChangeDirectory(cwd.Data());
169 // Check for setup script
170 if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
171 // Info("ParUtilities::SetupPAR", "Setting up for PAR %s", what);
172 gROOT->Macro("PROOF-INF/SETUP.C");
174 if (!gSystem->ChangeDirectory(cwd.Data())) return false;
178 //__________________________________________________________________
181 * @name PAR generation from script
184 * Service function to make a PAR out of a script.
186 * The script should contain can contain a sub-class of AliAnalysisTask.
187 * The script will be compiled on the slaves before loading the
188 * AliAnalysisManager. Parts to (not) be compiled can be protected like
192 * // This will _only_ be compiled in the servers
195 * // This will not be compiled in the servers
199 * @param script Script to upload and compile in the PAR
200 * @param deps Dependency pars
201 * @param isLocal Local build
203 * @return true on success.
205 static Bool_t MakeScriptPAR(Bool_t isLocal,
206 const TString& script,
210 TObjArray* depList = deps.Tokenize(", ");
212 // --- In local mode, just AcLic and load ------------------------
217 while ((dep = next()))
218 helper->LoadLibrary(dep->GetName());
221 if (gROOT->LoadMacro(Form("%s++g", script.Data())) < 0) {
222 Error("ParUtilities::MakeScriptPAR",
223 "Failed to build local library %s", script.Data());
229 // --- Get the base name -----------------------------------------
230 Info("ParUtilities::MakeScriptPAR", "Making par file for %s",
232 TString base(gSystem->BaseName(script));
233 Int_t idx = base.Last('.');
234 if (idx != kNPOS) base.Remove(idx);
236 // --- Check name of script file ---------------------------------
239 if (script.EndsWith(".C")) ext = "C";
240 else if (script.EndsWith(".cxx")) ext = "cxx";
241 else { ext = "C"; scr.Append(".C"); }
243 // --- Check if we can access the file ---------------------------
244 TString path = TString::Format(".:%s", TROOT::GetMacroPath());
245 char* loc = gSystem->Which(path, scr);
247 Error("ParUtilities::MakeScriptPAR",
248 "Script %s not found in %s", scr.Data(), path.Data());
253 // --- Create our temporary directory ----------------------------
254 TString tmpdir(gSystem->TempDirectory());
255 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
256 char* templ = new char[ltempl];
257 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
258 if (!mkdtemp(templ)) {
259 Error("ParUtilities::MakeScriptPAR",
260 "Failed to generate temporary directory from template %s",
265 Bool_t retVal = false;
267 // --- Make directories for package ------------------------------
268 TString dir = TString::Format("%s/%s", templ, base.Data());
269 // Set-up directories
270 if (gSystem->MakeDirectory(dir) < 0)
271 throw TString::Format("Could not make directory '%s'", base.Data());
272 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
273 throw TString::Format("Could not make directory %s/PROOF-INF",
276 // --- Copy the script to the setup directory --------------------
277 TString dest = TString::Format("%s/%s.%s", dir.Data(),
278 base.Data(), ext.Data());
279 Int_t ret = gSystem->CopyFile(full, dest, true);
281 case -1: throw TString::Format("Couldn't open %s for copy", scr.Data());
282 case -2: throw TString::Format("File %s exists", dest.Data());
283 case -3: throw TString::Format("Error while copying %s", scr.Data());
286 // --- Make scripts, etc. ----------------------------------------
287 if (!MakeScriptBuildScript(dir, base))
288 throw TString::Format("Failed to make build script");
289 if (!MakeScriptUtilityScript(dir))
290 throw TString::Format("Failed to make utility script");
291 if (!MakeScriptBuildMacro(dir, base, ext, depList))
292 throw TString::Format("Failed to make build macro");
293 if (!MakeScriptSetupMacro(dir, base, ext, depList))
294 throw TString::Format("Failed to setup macro");
296 // --- Pack up the archive ---------------------------------------
297 ret = gSystem->Exec(Form("(cd %s && tar -czf %s.par %s)",
298 templ, base.Data(),base.Data()));
300 throw TString::Format("Failed to create PAR file %s.PAR from %s",
301 base.Data(), dir.Data());
303 // --- Move PAR file to here -------------------------------------
304 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, base.Data(),
307 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
308 templ, base.Data(), base.Data(),
309 gSystem->GetError());
313 Error("ParUtilities::MakeScriptPAR", "%s", e.Data());
317 // --- Remove temporary directory --------------------------------
318 gSystem->Exec(Form("rm -rf %s", templ));
323 * Write a build script
325 * @param dir Directory to put it in
326 * @param base Base name
328 * @return true on success
330 static Bool_t MakeScriptBuildScript(const TString& dir,
333 // Make our build file
334 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
336 Error("ParUtilities::MakeScriptBuildScript",
337 "Failed to open out shell script");
341 << "if test x$ALICE_ROOT != x ; then\n"
342 << " if test x$ALICE_TARGET = x ; then\n"
343 << " export ALICE_TARGET=`$ROOTSYS/bin/root-config --arch`\n"
345 << " export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:"
346 << "${ALICE_ROOT}/lib/tgt_${ALICE_TARGET}\n"
348 << "echo BUILD.sh@`hostname`: Building " << base << "\n"
349 << "root.exe -l -out -q PROOF-INF/BUILD.C 2>&1 | tee "
351 << "echo BUILD.sh@`hostname`: done: $?\n"
354 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755) != 0) {
355 Error("ParUtilities::MakeScriptBuildScript",
356 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
363 * Write a build macro
365 * @param dir Directory to put macro in
366 * @param deps Dependencies
367 * @param base Base name of script to compile
368 * @param ext `extension' - last part of file name
370 * @return true on success
372 static Bool_t MakeScriptBuildMacro(const TString& dir,
375 const TCollection* deps) {
376 std::ofstream out(Form("%s/PROOF-INF/BUILD.C", dir.Data()));
378 Error("ParUtilities::MakeScriptBuildMacro","Failed to open build script");
381 out << "void BUILD() {\n"
382 << " gSystem->AddIncludePath(\"-DBUILD_PAR=1\");\n"
383 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
384 << " LoadROOTLibs();\n"
385 << " AddAliROOT();\n";
389 while ((dep = next())) {
390 out << " AddDep(\"" << dep->GetName() << "\");\t"
391 << " LoadDep(\"" << dep->GetName() << "\");\n";
394 out << " // gDebug = 5;\n"
395 << " int ret = gROOT->LoadMacro(\""
396 << base << "." << ext << "++g\");\n"
397 << " if (ret != 0) Fatal(\"BUILD\",\"Failed to build\");\n"
398 << " // else Info(\"BUILD\", \"Made " << base << "\");\n"
406 * Make a utility macro
408 * @param dir Directory to put the macro in
410 * @return true on success
412 static Bool_t MakeScriptUtilityScript(const TString& dir)
414 std::ofstream out(Form("%s/PROOF-INF/UTIL.C", dir.Data()));
416 Error("ParUtilities::MakeScriptUtilityScript",
417 "Failed to open utility script");
420 out << "void LoadROOTLibs() {\n"
421 << " gSystem->Load(\"libVMC\");\n"
422 << " gSystem->Load(\"libNet\");\n"
423 << " gSystem->Load(\"libTree\");\n"
424 << " gSystem->Load(\"libPhysics\");\n"
425 << " gSystem->Load(\"libMinuit\");\n"
427 << "void AddAliROOT() {\n"
428 << " TString val(gSystem->Getenv(\"ALICE_ROOT\"));\n"
429 << " if (val.IsNull())\n"
430 << " Warning(\"Add\",\"ALICE_ROOT not defined\");\n"
432 << " gSystem->AddIncludePath(Form(\"-I%s/include\",val.Data()));\n"
434 << "void AddDep(const char* env) {\n"
435 << " TString val(gSystem->Getenv(Form(\"%s_INCLUDE\",env)));\n"
436 << " if (val.IsNull())\n"
437 << " Warning(\"Add\",\"%s_INCLUDE not defined\", env);\n"
439 << " gSystem->AddIncludePath(Form(\"-I../%s\",val.Data()));\n"
442 << "void LoadDep(const char* name) {\n"
443 << " gSystem->AddDynamicPath(Form(\"../%s\",name));\n"
444 << " char* full = gSystem->DynamicPathName(name,true);\n"
446 << " full = gSystem->DynamicPathName(Form(\"lib%s\",name),true);\n"
448 << " full = gSystem->DynamicPathName(Form(\"lib%s.so\",name),true);\n"
450 << " Warning(\"LoadDep\",\"Module %s not found\", name);\n"
453 << " gSystem->Load(full);\n"
460 * Make a setup script
462 * @param dir Directory to put it in
463 * @param base Base name of target script
464 * @param ext Extension of target script
465 * @param deps Dependencies
467 * @return true on success
469 static Bool_t MakeScriptSetupMacro(const TString& dir,
472 const TCollection* deps)
474 // Make our set-up script
475 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
477 Error("ParUtilities::MakeScriptSetupMacro",
478 "Failed to open setup script");
481 out << "void SETUP() {\n"
482 << " gROOT->LoadMacro(\"PROOF-INF/UTIL.C\");\n"
483 << " LoadROOTLibs();\n"
484 << " // Info(\"SETUP\",\"Loading libraries\");\n";
488 while ((dep = next()))
489 out << " LoadDep(\"" << dep->GetName() << "\");\n";
491 out << " // gDebug = 5;\n"
492 << " // Info(\"SETUP\",\"Loading " << base <<"_"<< ext << ".so\");\n"
493 << " gSystem->Load(\"" << base << "_" << ext << ".so\");\n"
494 << " // gDebug = 0;\n"
495 << " gROOT->ProcessLine(\".include " << base << "\");\n"
496 << " gSystem->Setenv(\"" << base << "_INCLUDE\",\""
498 << " // Info(\"SETUP\", \"Done\");\n"
505 //__________________________________________________________________
508 * @name PAR generation from aux file list
510 static Bool_t MakeAuxFilePAR(const TList& files,
512 Bool_t verbose=false)
514 // --- Check input -----------------------------------------------
515 if (files.GetEntries() <= 0) return true;
517 // --- Create our temporary directory ----------------------------
518 Bool_t retval = true;
519 TString tmpdir(gSystem->TempDirectory());
520 int ltempl = tmpdir.Length() + 1 + 5 + 6 + 1;
521 char* templ = new char[ltempl];
522 snprintf(templ, ltempl, "%s/trainXXXXXX", tmpdir.Data());
523 if (!mkdtemp(templ)) {
524 Error("ParUtilities::MakeAuxFilePAR",
525 "Failed to generate temporary directory from template %s",
529 if (verbose) Printf("Preparing PAR file in %s", templ);
532 // --- Make directories for package ------------------------------
533 TString dir = TString::Format("%s/%s", templ, name.Data());
534 // Set-up directories
535 if (gSystem->MakeDirectory(dir) < 0)
536 throw TString::Format("Could not make directory '%s'", name.Data());
537 if (gSystem->MakeDirectory(Form("%s/PROOF-INF", dir.Data())))
538 throw TString::Format("Could not make directory %s/PROOF-INF",
543 while ((o = next())) {
544 TString fn(o->GetName());
545 if (verbose) Printf("Got %s", fn.Data());
546 if (fn.BeginsWith("/")) {
547 Warning("MakeAuxFilePAR", "Will not include absolute path %s",
549 continue; // absolute path
552 if (gSystem->AccessPathName(fn.Data())) {
553 Warning("MakeAuxFilePAR", "Cannot access %s", fn.Data());
554 continue; // non-exist
556 // Loop over path components and make directories as needed
557 TObjArray* comps = fn.Tokenize("/");
559 Int_t n = comps->GetEntriesFast();
560 if (verbose) Printf("Got %d path components in %s", n-1, fn.Data());
562 for (Int_t i = 0; i < n-1; i++) {
563 TObjString* comp = static_cast<TObjString*>(comps->At(i));
564 TString& c = comp->String();
565 if (c.IsNull()) continue;
566 if (c.EqualTo(".")) continue;
568 Bool_t doMake = true;
569 if (c.EqualTo("..")) { doMake = false; lvl--; }
571 cur = gSystem->ConcatFileName(cur, c);
573 Warning("MakeAuxFilePAR", "Path %s points outside archive, ignored",
580 if (!gSystem->AccessPathName(cur)) continue;
581 if (verbose) Printf("Making directory %s", cur.Data());
582 gSystem->MakeDirectory(cur);
585 if (verbose) Printf("cur=%s for %s lvl=%d", cur.Data(), fn.Data(), lvl);
587 if (lvl < 0) continue;
589 TString dest = TString::Format("%s/%s", cur.Data(),
590 gSystem->BaseName(fn.Data()));
591 if (verbose) Printf("%s -> %s", fn.Data(), dest.Data());
592 Int_t ret = gSystem->CopyFile(fn, dest, true);
594 case -1: throw TString::Format("Couldn't open %s for copy", fn.Data());
595 case -2: throw TString::Format("File %s exists", dest.Data());
596 case -3: throw TString::Format("Error while copying %s", fn.Data());
601 // Make our build file
602 if (verbose) Printf("Making build script");
603 std::ofstream out(Form("%s/PROOF-INF/BUILD.sh", dir.Data()));
605 Error("ParUtilities::MakeAuxFilePAR",
606 "Failed to open out shell script");
609 out << "#!/bin/sh\n\n"
610 << "echo \"Nothing to be done\"\n\n"
611 << "# EOF" << std::endl;
613 if (gSystem->Chmod(Form("%s/PROOF-INF/BUILD.sh", dir.Data()), 0755)) {
614 Error("ParUtilities::MakeAuxFilePAR",
615 "Failed to set exectuable flags on %s/PROOF-INF/BUILD.sh",
621 if (verbose) Printf("Making setup script");
622 // Make our setup file
623 std::ofstream out(Form("%s/PROOF-INF/SETUP.C", dir.Data()));
625 Error("ParUtilities::MakeAuxFilePAR",
626 "Failed to open out ROOT script");
629 out << "void SETUP()\n"
631 << " TString oldDir(gSystem->WorkingDirectory());\n"
632 << " TSystemDirectory* dir = new TSystemDirectory(\""
633 << name << "\",\"" << name << "\");\n"
634 << " TList* files = dir->GetListOfFiles();\n"
635 << " if (!gSystem->ChangeDirectory(oldDir)) {\n"
636 << " Error(\"SETUP\", \"Failed to go back to %s\",\n"
637 << " oldDir.Data());\n"
640 << " if (!files) {\n"
641 << " Warning(\"SETUP\", \"No files\");\n"
642 << " gSystem->Exec(\"pwd; ls -al\");\n"
645 << " files->Sort();\n"
646 << " TIter next(files);\n"
647 << " TSystemFile* file = 0;\n"
648 << " while ((file = static_cast<TSystemFile*>(next()))) {\n"
649 << " TString name(file->GetName());\n"
650 << " if (name == \".\" || name == \"..\") continue;\n"
651 << " TString title(file->GetTitle());\n"
652 << " TString full(gSystem->ConcatFileName(file->GetTitle(),\n"
653 << " name.Data()));\n"
654 << " if (file->IsA()->InheritsFrom(TSystemDirectory::Class()))\n"
655 << " full = title;\n"
656 << " gSystem->Symlink(full, name);\n"
659 << "// EOF " << std::endl;
662 if (verbose) Printf("Packing up");
664 ret = gSystem->Exec(Form("(cd %s && tar -c%szf %s.par %s)",
665 templ, (verbose ? "v" : ""),
666 name.Data(),name.Data()));
668 throw TString::Format("Failed to create PAR file %s.PAR from %s",
669 name.Data(), name.Data());
671 // --- Move PAR file to here -------------------------------------
672 if (verbose) Printf("Move here");
673 ret = gSystem->Exec(Form("mv -f %s/%s.par %s.par", templ, name.Data(),
676 throw TString::Format("Failed to rename %s/%s.par to %s.par: %s",
677 templ, name.Data(), name.Data(),
678 gSystem->GetError());
682 Printf("List content");
683 gSystem->Exec(Form("tar tzf %s.par", name.Data()));
688 Error("ParUtilities::MakeAuxFilePAR", "%s", e.Data());
692 // --- Remove temporary directory --------------------------------
693 gSystem->Exec(Form("rm -rf %s", templ));