#!/bin/bash
#include benchmark.config
+# blame: Mikolaj Krzewicki, mkrzewic@cern.ch
# this script runs the CPass0/CPass1 train
# produced OCDB updates are local
return 1
fi
- #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
runpath=${outputDir}
- #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
+ [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
mkdir -p ${runpath}
if [[ ! -d ${runpath} ]]; then
touch ${doneFileTmp}
#runCPassX/C expects the raw chunk to be linked in the run dir
#despite it being accessed by the full path
- ln -s ${infile} ${runpath}/${chunkName}
+ if [[ $copyInputData == 0 ]]; then
+ ln -s ${infile} ${runpath}/${chunkName}
+ else
+ copyFileToLocal ${infile} ${runpath}/${chunkName}
+ fi
#####MC
if [[ -n ${generateMC} ]]; then
fi
######
- if [[ ! -f ${inputList} && -z ${pretend} ]]; then
+ if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
touch ${doneFileTmp}
echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
fi
#run CPass0
- echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
+ echo "${runpath}/runCPass0.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
if [[ -n ${pretend} ]]; then
sleep ${pretendDelay}
touch AliESDs.root
touch rec.log
touch calib.log
else
- echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
- ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
+ #caveat: in the local case, first arg must start with a slash
+ ./runCPass0.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fi
#move stuff to final destination
# [dberzano] OK this is fine!
echo rm -f ./${chunkName}
rm -f ./${chunkName}
- echo "cp -R ${runpath}/* ${outputDir}"
- cp -p -R ${runpath}/* ${outputDir}
+ echo "paranoidCp ${runpath}/* ${outputDir}"
+ paranoidCp ${runpath}/* ${outputDir}
echo
#validate CPass0
cd ${outputDir}
- touch ${doneFileTmp}
- echo "dir ${outputDir}" >> ${doneFileTmp}
if summarizeLogs >> ${doneFileTmp}; then
[[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
[[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
#Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
export PRODUCTION_METADATA="OutputDir=cpass1"
- if [[ ! -f ${inputList} && -z ${pretend} ]]; then
+ if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
touch ${doneFileTmp}
echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
return 1
fi
- #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
runpath=${outputDir}
- #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
+ [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
#MC
if [[ "${infile}" =~ galice\.root ]]; then
fi
#this is needed for runCPass1.sh
- ln -s ${infile} ${runpath}/${chunkName}
+ if [[ $copyInputData == 0 ]]; then
+ ln -s ${infile} ${runpath}/${chunkName}
+ else
+ copyFileToLocal ${infile} ${runpath}/${chunkName}
+ fi
logOutputDir=${runpath}
[[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
#run CPass1
chmod u+x runCPass1.sh
- echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
+ echo "${runpath}/runCPass1.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
if [[ -n ${pretend} ]]; then
sleep ${pretendDelay}
touch AliESDs_Barrel.root
touch qa.log
touch filtering.log FilterEvents_Trees.root
else
- ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
+ #caveat: in the local case, first arg must start with a slash
+ ./runCPass1.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
[[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
[[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
/bin/ls
echo rm -f ./${chunkName}
rm -f ./${chunkName}
- echo "cp -R ${runpath}/* ${outputDir}"
- cp -pf -R ${runpath}/* ${outputDir}
+ echo "paranoidCp ${runpath}/* ${outputDir}"
+ paranoidCp ${runpath}/* ${outputDir}
echo
#validate CPass1
cd ${outputDir}
- touch ${doneFileTmp}
- echo "dir ${outputDir}" >> ${doneFileTmp}
if summarizeLogs >> ${doneFileTmp}; then
[[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
[[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
- #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
runpath=${outputDir}
- #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
+ [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
mkdir -p ${runpath}
if [[ ! -d ${runpath} ]]; then
/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fi
- echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
+ echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
if [[ -n ${pretend} ]]; then
sleep ${pretendDelay}
touch CalibObjects.root
echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
else
- ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
+ ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy >> "mergeMakeOCDB.log"
#produce the calib trees for expert QA (dcsTime.root)
goMakeLocalOCDBaccessConfig ./OCDB
/bin/ls
#copy all to output dir
- cp -pf -R ${runpath}/* ${outputDir}
+ echo "paranoidCp ${runpath}/* ${outputDir}"
+ paranoidCp ${runpath}/* ${outputDir}
if [[ -n ${generateMC} ]]; then
goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
#validate merging cpass0
cd ${outputDir}
- touch ${doneFileTmp}
- echo "dir ${outputDir}" >> ${doneFileTmp}
if summarizeLogs >> ${doneFileTmp}; then
[[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
[[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
- #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
runpath=${outputDir}
- #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
+ [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
mkdir -p ${runpath}
if [[ ! -d ${runpath} ]]; then
echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fi
-
- echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
+
+ echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
if [[ -n ${pretend} ]]; then
sleep ${pretendDelay}
touch ocdb.log
touch ${qaMergedOutputFileName}
mkdir -p OCDB
else
- ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
+ ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy
#merge QA (and filtered trees)
[[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
/bin/ls
#copy all to output dir
- cp -pf -R ${runpath}/* ${outputDir}
+ echo "paranoidCp ${runpath}/* ${outputDir}"
+ paranoidCp ${runpath}/* ${outputDir}
#validate merge cpass1
cd ${outputDir}
- touch ${doneFileTmp}
- echo "dir ${outputDir}" >> ${doneFileTmp}
if summarizeLogs >> ${doneFileTmp}; then
[[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
[[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
#if which greadlink; then self=$(greadlink -f "${0}"); fi
#for reference copy the setup to the output dir
- cp ${self} ${commonOutputPath}
- cp ${configFile} ${commonOutputPath}
- cp ${inputFileList} ${commonOutputPath}
+ paranoidCp ${self} ${commonOutputPath}
+ paranoidCp ${configFile} ${commonOutputPath}
+ paranoidCp ${inputFileList} ${commonOutputPath}
#submit - use makeflow if available, fall back to old stuff when makeflow not there
if which makeflow; then
else
echo "no makeflow!"
fi
+
+ #summarize the run based on the makeflow log
+ #and add it to the end of summary log
+ awk '/STARTED/ {startTime=$3}
+ /COMPLETED/ {endTime=$3}
+ END {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
+ benchmark.makeflow.makeflowlog | tee -a summary.log
+ paranoidCp summary.log ${commonOutputPath}
+
return 0
}
"stderr"
)
+ #put dir information in the output
+ echo "dir $PWD"
+
#check logs
local logstatus=0
for log in ${logFiles[*]}; do
fi
pwd
/bin/ls
- touch ${doneFile}
summarizeLogs >> ${doneFile}
#echo mv -f * ${outputDir}
echo "submit make a summary"
echo
- submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
+ [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
+ submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
LASTJOB=${JOBID6}
#################################################################################
stackTraceTree()
(
- # make stacktrace processing in case of standard root crash log
- # input is a (list of) text files with the stack trace (either gdb aoutput
- # produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core, or the root crash log), output is a TTree formatted table.
-# example usage:
-# benchmark.sh stackTraceTree /foo/*/rec.log
+ if [[ $# -lt 1 ]]; then
+ echo 'make stacktrace processing in case of standard root crash log'
+ echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
+ echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
+ echo 'or the root crash log), output is a TTree formatted table.'
+ echo 'example usage:'
+ echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
+ echo 'benchmark.sh stackTraceTree $(cat file.list)'
+ echo 'benchmark.sh stackTraceTree `cat file.list`'
+ return 0
+ fi
gawk '
BEGIN {
- print "frame/I:method/C:line/C:cpass/I:aliroot/I";
+ print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
RS="#[0-9]*";
aliroot=0;
read=1;
read==1 {
if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
gsub("#","",RT);
- if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
+ if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
}
' "$@" 2>/dev/null
)
echo per run stats:
/bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
do
- dir=$(goPrintValues calibfile - ${x})
+ dir=$(goPrintValues dir - ${x})
runNumber=$(guessRunNumber ${dir})
[[ -z ${runNumber} ]] && continue
cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
#copy output files
- cp -r QAplots ${commonOutputPath}
- cp *.list ${commonOutputPath}
- cp *.root ${commonOutputPath}
- cp *.log ${commonOutputPath}
+ exec &> >(tee fileCopy.log)
+ paranoidCp QAplots ${commonOutputPath}
+ paranoidCp *.list ${commonOutputPath}
+ paranoidCp *.root ${commonOutputPath}
+ paranoidCp *.log ${commonOutputPath}
+ paranoidCp fileCopy.log ${commonOutputPath}
return 0
)
logToFinalDestination=1
ALIROOT_FORCE_COREDUMP=1
pretendDelay=0
+ copyInputData=0
#first, source the config file
if [ -f ${configFile} ]; then
return 0
}
+copyFileToLocal()
+(
+ #copies a file from either a remote or local location to a local destination
+ src="$1"
+ dst="$2"
+
+ proto="${src%%://*}"
+ if [[ "$proto" == "$src" ]]; then
+ cp "$src" "$dst"
+ else
+ case "$proto" in
+ root)
+ xrdcp -f "$src" "$dst"
+ ;;
+ http)
+ curl -L "$src" -O "$dst"
+ ;;
+ *)
+ echo "protocol not supported: $proto"
+ return 1
+ ;;
+ esac
+ fi
+)
+
+paranoidCp()
+(
+ #recursively copy files and directories
+ #to avoid using find and the like as they kill
+ #the performance on some cluster file systems
+ #does not copy links to avoid problems
+ sourceFiles=("${@}")
+ destination="${sourceFiles[@]:(-1)}" #last element
+ unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
+ for src in "${sourceFiles[@]}"; do
+ if [[ -f "${src}" && ! -h "${src}" ]]; then
+ paranoidCopyFile "${src}" "${destination}"
+ elif [[ -d "${src}" && ! -h "${src}" ]]; then
+ src="${src%/}"
+ dst="${destination}/${src##*/}"
+ mkdir -p "${dst}"
+ paranoidCp "${src}"/* "${dst}"
+ fi
+ done
+)
+
+paranoidCopyFile()
+(
+ #copy a single file to a target in an existing dir
+ #repeat a few times if copy fails
+ src="${1}"
+ dst="${2}"
+ [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
+ [[ -z "${maxCopyTries}" ]] && maxCopyTries=5
+ #echo "maxCopyTries=${maxCopyTries}"
+ echo "cp ${src} ${dst}"
+ cp "${src}" "${dst}"
+ i=0
+ until cmp -s "${src}" "${dst}"; do
+ echo "try: ${i}"
+ [[ -f "${dst}" ]] && rm "${dst}"
+ cp "${src}" "${dst}"
+ [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1
+ (( i++ ))
+ done
+ return 0
+)
+
guessRunData()
{
#guess the period from the path, pick the rightmost one
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff