*
* @return true on success
*/
-Bool_t GetOne(const TString& base, const TString& dir)
+Bool_t GetOne(const TString& base, const TString& dir, Bool_t unpack)
{
TString src(gSystem->ConcatFileName(base,dir));
src = gSystem->ConcatFileName(src,"root_archive.zip");
+ TString name;
+ name.Form("root_archive_%s",dir.Data());
TString dest;
- dest.Form("root_archive_%s.zip",dir.Data());
+ dest.Form("%s.zip",name.Data());
if (!TFile::Cp(src, dest)) {
Error("GetOne","Failed to download %s -> %s",
src.Data(), dest.Data());
return false;
}
+ if (!unpack) return true;
+ gSystem->Exec(Form("mkdir -p %s && (cd %s && unzip ../%s)",
+ name.Data(), name.Data(), dest.Data()));
return true;
}
-void GridDownload(const TString& base, const TString& runs)
+void GridDownload(const TString& base, const TString& runs, Bool_t unpack)
{
gEnv->SetValue("XSec.GSI.DelegProxy", "2");
if (!TGrid::Connect("alien://")) {
TObjString* run = 0;
TIter next(runArray);
while ((run = static_cast<TObjString*>(next()))) {
- GetOne(base, run->String());
+ GetOne(base, run->String(), unpack);
}
}
// EOF
* @return Formatted string
* @ingroup pwglf_forward_trains_helper
*/
-TString TokenName(const TString& name,
- const TString& ext,
- Bool_t merge=false)
+TString CacheFileName(const TString& name,
+ const TString& ext,
+ Bool_t merge=false)
{
return TString::Format("%s%s.%s", name.Data(),
(merge ? "_merge" : ""), ext.Data());
* @return true if file exits
* @ingroup pwglf_forward_trains_helper
*/
-Bool_t CheckTokens(const TString& name,
- const TString& ext,
- Bool_t merge=false)
+Bool_t CheckCacheFile(const TString& name,
+ const TString& ext,
+ Bool_t merge=false)
{
// TSystem::AccessPathName return false if file is there
- return !gSystem->AccessPathName(TokenName(name, ext, merge));
+ return !gSystem->AccessPathName(CacheFileName(name, ext, merge));
}
/**
* Remove a token file
* @param merge Merging stage or not
* @ingroup pwglf_forward_trains_helper
*/
-void RemoveTokens(const TString& name,
+void RemoveCacheFile(const TString& name,
const TString& ext,
Bool_t merge=false)
{
- gSystem->Unlink(TokenName(name, ext, merge));
+ gSystem->Unlink(CacheFileName(name, ext, merge));
}
/**
*
* @ingroup pwglf_forward_trains_helper
*/
-TObjArray* ReadTokens(const TString& name,
+TObjArray* ReadCacheFile(const TString& name,
const TString& ext,
bool merge=false)
{
(merge ? "_merge" : ""), ext.Data());
std::ifstream in(fn.Data());
if (!in) {
- Error("ReadTokens", "Failed to open %s", fn.Data());
+ Error("ReadCacheFile", "Failed to open %s", fn.Data());
return 0;
}
TString ln;
*/
TObjArray* ReadJobIDs(const TString& name, bool merge=false)
{
- return ReadTokens(name, "jobid", merge);
+ return ReadCacheFile(name, "jobid", merge);
}
/**
*/
TObjArray* ReadStages(const TString& name, bool merge=false)
{
- return ReadTokens(name, "stage", merge);
+ return ReadCacheFile(name, "stage", merge);
}
/**
return true;
}
+/**
+ * Check if the AliEn token is valid
+ *
+ *
+ * @return true if it is
+ */
+Bool_t CheckAlienToken()
+{
+ Int_t ret = gSystem->Exec("alien-token-info > /dev/null 2>&1");
+ if (ret != 0) {
+ Printf("=== AliEn token not valid");
+ return false;
+ }
+ return true;
+}
/**
* Refersh the grid token every 6th hour
* @param now
* @param force
*/
-void RefreshToken(UInt_t now, Bool_t force=false)
+#if 0
+void RefreshAlienToken(UInt_t, Bool_t f=false)
+{}
+#else
+void RefreshAlienToken(UInt_t now, Bool_t force=false)
{
- static UInt_t start = 0;
- if (start == 0) start = now;
-
- // Try to refresh token every 6th hour
- if (!force || (now - start) / 60 / 60 < 6) return;
+ Bool_t renew = force;
+ if (!renew && !CheckAlienToken()) renew = true;
+
+ if (!renew) {
+ TString l = gSystem->GetFromPipe(Form("cat /tmp/gclient_token_%d",
+ gSystem->GetUid()));
+ TObjArray* lines = l.Tokenize("\n");
+ TObjString* sline = 0;
+ UInt_t expire = 0;
+ TIter next(lines);
+ while ((sline = static_cast<TObjString*>(next()))) {
+ TString& line = sline->String();
+ if (!line.BeginsWith("Expiretime")) continue;
+
+ Size_t eq = line.Index("=");
+ TString sdatime = line(eq+2, line.Length()-eq-2);
+ expire = sdatime.Atoi();
+ break;
+ }
+ lines->Delete();
+ // If the expiration date/time has passed or is less than 30 min
+ // away, we refresh
+ Int_t diff = (expire - now);
+ if (now > expire || diff < 30*60) renew = true;
+
+ Printf("=== Now: %d, Expires: %d, in %03d:%02d:%02d -> %s",
+ now, expire, diff/60/60, (diff/60 % 60), (diff % 60),
+ (renew ? "renew" : "nothing"));
+
+ }
+
+ if (!renew) return;
// Reset the start time
- start = now;
Printf("=== Refreshing AliEn token");
gSystem->Exec("alien-token-init");
Printf("=== Done refreshing AliEn token");
}
+#endif
+
+
/**
* Wait of jobs to finish
*
Int_t delay,
Bool_t batch)
{
+ if (!CheckAlienToken()) return false;
// Bool_t stopped = false;
TFileHandler h(0, 0x1);
- RefreshToken(0, true);
+ // RefreshAlienToken(0, true);
do {
Bool_t allDone = true;
TDatime t;
Printf(" %d(%s)=%s", job, stages->At(i)->GetName(), state.Data());
}
- RefreshToken(now);
+ RefreshAlienToken(now);
if (allDone) break;
if (missing >= total) {
*/
void GridWatch(const TString& name, Bool_t batch=false, UShort_t delay=5*60)
{
+#if 1
+ // We use command line tools instead of ROOT interface - which is
+ // broken so badly that it's hard to believe it ever worked.
gEnv->SetValue("XSec.GSI.DelegProxy", "2");
TGrid::Connect("alien:///");
if (!gGrid) {
Error("GridWatch", "Failed to connect to the Grid");
return;
}
+#endif
TObjArray* jobIDs = ReadJobIDs(name, false);
TObjArray* stages = ReadStages(name, false);
if (!ParseJobIDs(jobIDs, jobs)) return;
gSystem->Sleep(10*1000);
- if (!(CheckTokens(name, "jobid", true) &&
- CheckTokens(name, "stage", true)))
- WaitForJobs(jobs, stages, delay, batch);
+ if (!(CheckCacheFile(name, "jobid", true) &&
+ CheckCacheFile(name, "stage", true)))
+ if (!WaitForJobs(jobs, stages, delay, batch)) return;
delete jobIDs;
delete stages;
// return;
do {
- if (!CheckTokens(name, "jobid", true) &&
- !CheckTokens(name, "stage", true)) {
+ if (!CheckCacheFile(name, "jobid", true) &&
+ !CheckCacheFile(name, "stage", true)) {
+ if (!CheckAlienToken()) return;
Printf("Now executing terminate");
gSystem->Exec("aliroot -l -b -q Terminate.C");
gSystem->Sleep(10*1000);
}
- Printf("Reading job ids");
+ Printf("Reading job ids");
jobIDs = ReadJobIDs(name, true);
stages = ReadStages(name, true);
if (!ParseJobIDs(jobIDs, jobs)) {
Error("GridWatch", "Failed to parse job ids %s",
- TokenName(name,"jobid",true).Data());
+ CacheFileName(name,"jobid",true).Data());
return;
}
- WaitForJobs(jobs, stages, delay, batch);
+ if (!WaitForJobs(jobs, stages, delay, batch)) return;
Bool_t allFinal = true;
for (Int_t i = 0; i < jobs.GetSize(); i++) {
Printf("All jobs in final stage");
if (allFinal) break;
- RemoveTokens(name, "jobid", true);
- RemoveTokens(name, "stage", true);
+ RemoveCacheFile(name, "jobid", true);
+ RemoveCacheFile(name, "stage", true);
} while (true);
Printf("Finished");
SaveSetupShell("rerun", ClassName(), fName, tmp, uopts);
SaveSetupROOT("ReRun", ClassName(), fName, tmp, uopts);
if (fHelper) fHelper->AuxSave(fEscapedName, asShellScript);
+ SavePostShellScript();
}
/**
* Save a setup as a shell script
<< "//" << std::endl;
o.close();
}
+ /**
+ * Write shell code to do post processing after terminate. This
+ * code should deal with a single run (or run range). The following
+ * shell variables are available to the code:
+ *
+ * - @c $prefix Relative path to job directory or empty
+ * - @c $dest Destination for output to be stored
+ *
+ * Note, the code is injected into a shell function, and should
+ * therefor not define new functions or the like.
+ *
+ * @param o The output stream.
+ */
+ virtual void PostShellCode(std::ostream& o)
+ {
+ o << " echo \"Nothing to do for " << ClassName()
+ << " train\"" << std::endl;
+ }
+ /**
+ * Save a script to do post processing after terminate on each run
+ * or run-range.
+ *
+ * The shell script will execute the train defined code (in
+ * PostShellCode) for each run or run-range. The train defined code
+ * and call drawing and summarizing macros or the like.
+ *
+ * In case of Grid analysis, this script will download and extract
+ * the appropriate ZIP files to separate directories, and then
+ * change directory to these directories and execute the train
+ * defined shell code there. In this case, the script defines the
+ * shell variable @c $prefix as the relative path to the job
+ * directory.
+ *
+ */
+ void SavePostShellScript()
+ {
+ std::ofstream f("post.sh");
+ if (!f) {
+ Error("SavePostAll", "Failed to open post.sh script");
+ return;
+ }
+ f << "#!/bin/bash\n"
+ << "# Generated by " << ClassName() << "\n"
+ << "set -e\n"
+ << "\n"
+ << "dest=$1\n"
+ << "prefix=\n"
+ << "\n"
+ << "doall() {"
+ << std::endl;
+ PostShellCode(f);
+ f << "}\n"
+ << "\n"
+ << "if test ! -f Download.C ;then\n"
+ << " doall\n"
+ << " exit\n"
+ << "fi\n"
+ << "\n"
+ << "if test ! -f .download ; then\n"
+ << " aliroot -l -b -q Download.C\\(1\\)\n"
+ << " touch .download\n"
+ << "fi\n"
+ << "prefix=../\n"
+ << "\n"
+ << "for i in root_archive_*.zip ; do\n"
+ << " d=`basename $i .zip` \n"
+ << " if test ! -d $d ; then\n"
+ << " echo \"Directory $d missing\"\n"
+ << " continue\n"
+ << " fi\n"
+ << " \n"
+ << " (cd $d && doall)\n"
+ << "done\n"
+ << "# EOF"
+ << std::endl;
+ f.close();
+ gSystem->Exec("chmod a+x post.sh");
+ }
+
/* @} */
TString fName;
TString fEscapedName;
git://git.uio.no / u / mrichter / AliRoot.git / commitdiff