echo "uses a batch system (SGE):"
echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
echo "extraOpts if non-empty override the config file, e.g.:"
- echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10"
+ echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
fi
return
fi
"WaitForOutput") goWaitForOutput "$@";;
"Merge") goMerge "$@";;
*)
- eval ${runMode} $@
+ ${runMode} "$@"
;;
esac
}
runNumber=${6}
jobindex=${7}
shift 7
- parseConfig ${configFile} $@
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
#use the jobindex only if set and non-negative
if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
#Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
export PRODUCTION_METADATA="OutputDir=cpass0"
- #runCPassX/C expects the raw chunk to be linked in the run dir
- #despite it being accessed by the full path
+ if [[ "${inputList}" =~ \.root$ ]]; then
+ infile=${inputList}
+ else
+ infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
+ fi
chunkName=${infile##*/}
- ln -s ${infile} ${runpath}/${chunkName}
outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
mkdir -p ${outputDir}
runpath=${outputDir}
[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
[[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
-
mkdir -p ${runpath}
[[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
cd ${runpath}
- if [[ "${inputList}" =~ \.root$ ]]; then
- infile=${inputList}
- else
- infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
- fi
+ #runCPassX/C expects the raw chunk to be linked in the run dir
+ #despite it being accessed by the full path
+ ln -s ${infile} ${runpath}/${chunkName}
#####MC
if [[ -n ${generateMC} ]]; then
#validate CPass0
cd ${outputDir}
touch ${doneFile}
- [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
- [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
- [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
echo "dir ${outputDir}" >> ${doneFile}
- summarizeLogs >> ${doneFile}
+ if summarizeLogs >> ${doneFile}; then
+ [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
+ [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
+ [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
+ fi
[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
+ return 0
}
goCPass1()
runNumber=${6}
jobindex=${7}
shift 7
- parseConfig ${configFile} $@
+ extraOpts=("$@")
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
#use the jobindex only if set and non-negative
if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
else
infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fi
-
chunkName=${infile##*/}
+
outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
mkdir -p ${outputDir}
[[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
ln -s ../OCDB Barrel/OCDB
ln -s ../OCDB OuterDet/OCDB
+ #setup the filtering
+ #the following option enables the filtering task inside the QAtrain_duo.C
+ [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
+ #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
+ if [[ -n ${filteringFactorHighPt} ]]; then
+ export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
+ fi
+ if [[ -n ${filteringFactorV0s} ]]; then
+ export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
+ fi
+
#run CPass1
chmod u+x runCPass1.sh
echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
touch rec.log
touch calib.log
touch qa.log
+ touch filtering.log FilterEvents_Trees.root
else
./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
+
+ [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
+ [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
+ [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
+ [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
+ [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
+ [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
+ [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
+ [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
+
+ #make the filtered tree (if requested and not already produced by QA
+ [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
+ if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
+ goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
+ fi
+
fi
##handle possible crashes in QA (happens often in trunk)
# time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
# cd ../
#fi
- [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
- [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
- [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
- [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
- [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
- [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
- [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
#move stuff to final destination
echo "this directory (${PWD}) contents:"
#validate CPass1
cd ${outputDir}
touch ${doneFile}
- [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
- [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
- [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
- [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
echo "dir ${outputDir}" >> ${doneFile}
- summarizeLogs >> ${doneFile}
-
+ if summarizeLogs >> ${doneFile}; then
+ [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
+ [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
+ [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
+ [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
+ [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
+ fi
+
[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
+ return 0
}
runNumber=${4}
calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
shift 5
- parseConfig ${configFile} $@
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
[[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
#validate merging cpass0
cd ${outputDir}
touch ${doneFile}
- [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
- [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
echo "dir ${outputDir}" >> ${doneFile}
- summarizeLogs >> ${doneFile}
+ if summarizeLogs >> ${doneFile}; then
+ [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
+ [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
+ fi
[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
+ return 0
}
goMergeCPass1()
runNumber=${4}
calibrationFilesToMerge=${5}
qaFilesToMerge=${6}
- shift 6
- parseConfig ${configFile} $@
+ filteredFilesToMerge=${7}
+ shift 7
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
[[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
#merge calibration
chmod u+x ${mergingScript}
mkdir -p OCDB
+
+ #if not provided, create the lists of files to merge
+ if [[ ! -f ${filteredFilesToMerge} ]]; then
+ echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
+ /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
+ fi
if [[ ! -f ${calibrationFilesToMerge} ]]; then
- echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}"
- /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge}
+ echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
+ /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
+ fi
+ if [[ ! -f ${qaFilesToMerge} ]]; then
+ #find the files, but only store the directories (QAtrain_duo.C requires this)
+ echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
+ /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fi
echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
- #merge QA
+ #merge QA (and filtered trees)
[[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
[[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
- if [[ ! -f ${qaFilesToMerge} ]]; then
- #find the files, but only store the directories (QAtrain_duo.C requires this)
- echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
- eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
- fi
-
- #do the merge
#echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
if [[ -n ${pretend} ]]; then
touch ${qaMergedOutputFileName}
touch merge.log
touch trending.root
+ touch FilterEvents_Trees.root
+ touch CalibObjects.root
+ touch dcsTime.root
+ touch ${qaMergedOutputFileName}
else
#aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
mv QAresults_barrel.root ${qaMergedOutputFileName}
mv trending_barrel.root trending.root
+
+ #merge filtered trees
+ echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
+ aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
fi
#produce the calib trees for expert QA
#validate merge cpass1
cd ${outputDir}
touch ${doneFile}
- [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
- [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
- [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
- [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
echo "dir ${outputDir}" >> ${doneFile}
- summarizeLogs >> ${doneFile}
+ if summarizeLogs >> ${doneFile}; then
+ [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
+ [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
+ [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
+ [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
+ [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
+ fi
[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
+ return 0
}
goMerge()
outputFile=${2}
configFile=${3-"becnhmark.config"}
shift 3
- parseConfig ${configFile} $@
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
[[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
[[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
rm -f ${outputFile}
aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
+ return 0
}
goSubmitMakeflow()
inputFileList=${2}
configFile=${3}
shift 3
- extraOpts="$@"
- parseConfig ${configFile} ${extraOpts}
+ extraOpts=("$@")
+ if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
[[ -z ${configFile} ]] && configFile="benchmark.config"
[[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
#submit - use makeflow if available, fall back to old stuff when makeflow not there
if which makeflow; then
- goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow
+ goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" > benchmark.makeflow
makeflow ${makeflowOptions} benchmark.makeflow
cd ../
else
echo "no makeflow!"
fi
+ return 0
}
goGenerateMakeflow()
{
#generate the makeflow file
+ [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
productionID=${1}
inputFileList=${2}
configFile=${3}
shift 3
- extraOpts="$@"
- parseConfig ${configFile} ${extraOpts}
+ extraOpts=("$@")
+ if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
[[ -z ${configFile} ]] && configFile="benchmark.config"
[[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
commonOutputPath=${baseOutputDirectory}/${productionID}
+ self=$(readlink -f "${0}")
+ echo "self: $self"
#these files will be made a dependency - will be copied to the working dir of the jobs
declare -a copyFiles
inputFiles=(
#create the makeflow file
[[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
- declare -a arr_cpass1_final
- listOfRuns[0]=${runNumber}
- [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq))
- runindex=0
- for runNumber in ${listOfRuns[*]}; do
+ declare -A arr_cpass0_merged arr_cpass1_merged
+ declare -A arr_cpass0_calib_list arr_cpass1_calib_list
+ declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
+ declare -A arr_cpass0_profiled_outputs
+ declare -A listOfRuns
+ [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
+ while read x; do listOfRuns[$(guessRunNumber ${x})]=1; done < ${inputFileList}
+ for runNumber in "${!listOfRuns[@]}"; do
[[ -z ${runNumber} ]] && continue
[[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
- unset arr_cpass0_outputs_onerun
- unset arr_cpass1_outputs_onerun
- declare -a arr_cpass0_outputs_onerun
- declare -a arr_cpass1_outputs_onerun
+ unset arr_cpass0_outputs
+ unset arr_cpass1_outputs
+ declare -a arr_cpass0_outputs
+ declare -a arr_cpass1_outputs
jobindex=0
+ inputFile=""
while read inputFile; do
currentDefaultOCDB=${defaultOCDB}
[[ -z ${autoOCDB} ]] && autoOCDB=1
fi
#CPass0
- arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
- echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
- echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
+ arr_cpass0_outputs[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
+ echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
+ echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
+ for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
echo
#CPass1
- arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
- echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
- echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
+ arr_cpass1_outputs[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
+ echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
+ echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
+ for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
echo
((jobindex++))
done< <(grep "/000${runNumber}/" ${inputFileList})
#CPass0 list of Calib files to merge
- arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list"
- echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
- echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}"
+ arr_cpass0_calib_list[${runNumber}]="meta/cpass0.calib.run${runNumber}.list"
+ echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
+ echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
echo
#CPass0 merging
- arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done"
- echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}"
- echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}"
+ arr_cpass0_merged[${runNumber}]="meta/merge.cpass0.run${runNumber}.done"
+ echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
+ echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
+ for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
echo
- #CPass1 list of Calib/QA files to merge
- # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
- # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
+ #CPass1 list of Calib/QA/ESD/filtered files
+ # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
+ # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
# this corresponds to the last merging stage)
- arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list"
- echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
- echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};"
+ arr_cpass1_QA_list[${runNumber}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
+ echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
+ echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
echo
- arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
- echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
- echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
+ arr_cpass1_calib_list[${runNumber}]="meta/cpass1.calib.run${runNumber}.list"
+ echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
+ echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
echo
-
+ arr_cpass1_ESD_list[${runNumber}]="meta/cpass1.ESD.run${runNumber}.list"
+ echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
+ echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
+ echo
+ arr_cpass1_filtered_list[${runNumber}]="meta/cpass1.filtered.run${runNumber}.list"
+ echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
+ echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
+ echo
+
#CPass1 merging
- arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done"
- echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}"
- echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}"
+ arr_cpass1_merged[${runNumber}]="meta/merge.cpass1.run${runNumber}.done"
+ echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
+ echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
+ for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
echo
- arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list"
- echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
- echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
-
- if [ ${runESDfiltering} -eq 1 ]; then
- arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done"
- echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}"
- echo " ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
+ #CPass0 wrapped in a profiling tool (valgrind,....)
+ if [[ -n profilingCommand ]]; then
+ arr_cpass0_profiled_outputs[${runNumber}]="meta/cpass0.jobProfiled.run${runNumber}.done"
+ echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
+ echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
+ for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done;
+ echo "\"useProfilingCommand=${profilingCommand}\""
+ echo
fi
- ((runindex++))
- done
+ done #runs
#Summary
- echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}"
+ echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
echo
+ return 0
}
goPrintValues()
[[ ${outputFile} =~ "-" ]] && outputFile=""
shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
+ return 0
}
goCreateQAplots()
outputDir=${3}
configFile=${4}
shift 4
- parseConfig ${configFile} ${@}
+ if ! parseConfig ${configFile} ${@}; then return 1; fi
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
[[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
- ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
+ ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
cd ${olddir}
+ return 0
}
goTest()
exec > >(tee test.log)
echo "$@"
echo something
+ return 0
}
alirootInfo()
echo ""
git diff ${currentBranch}
popd
+ return 0
}
setYear()
local path=${2}
[[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
echo ${path}
+ return 0
}
guessPeriod()
[[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
done
echo ${period}
+ return 0
}
guessYear()
[[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
done
echo ${year}
+ return 0
}
guessRunNumber()
[[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
done
echo ${runNumber}
+ return 0
}
validateLog()
fi
elif [[ ${validationStatus} -eq 1 ]]; then
echo "${finallog} BAD ${errorSummary}"
+ logstatus=1
elif [[ ${validationStatus} -eq 2 ]]; then
echo "${finallog} OK MWAH ${errorSummary}"
fi
echo ${tmp%/*} | \
awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
done
+ return 0
}
goMakeLocalOCDBaccessConfig()
echo "!!!!!!! CPass0 produced no OCDB entries"
return 1
fi
+ return 0
}
goMakeFilteredTrees()
configFile=${11-"benchmark.config"}
esdFileName=${12-"AliESDs_Barrel.root"}
shift 12
- parseConfig ${configFile} $@
+ if ! parseConfig ${configFile} "$@"; then return 1; fi
commonOutputPath=${PWD}
doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
- runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
+ runpath=${outputDir}
+ [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
+ [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
+
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
cd ${commonOutputPath}
[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
+ return 0
}
submit()
{
umask 0002
- [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1
+ [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
JobID=${1}
startID=${2}
endID=${3}
waitForJOBID=${4}
command=${5}
- commandArgs=${6}
+ shift 5
+ commandArgs=("$@")
+
+ #add quote strings around the extra arguments
+ for ((i=0; i<${#commandArgs[@]}; i++)); do
+ commandArgs[i]=\"${commandArgs[i]}\"
+ done
[[ -z ${waitForJOBID} ]] && waitForJOBID=0
else
#new SGE farm
if [[ ${waitForJOBID} =~ "000" ]]; then
- echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
- ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
+ echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
+ ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
else
- echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
- ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
+ echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
+ ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
fi
fi
+ return 0
}
goSubmitBatch()
configFile=${3:-"benchmark.config"}
configFile=$(readlink -f ${configFile})
shift 3
- extraOpts="$@"
- parseConfig ${configFile} ${extraOpts}
+ extraOpts=("$@")
+ if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
#redirect all output to submit.log
echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
echo "Using Config File: '${configFile}'"
fi
- # and print the configuration
- [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
+ [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
#move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
self=$(readlink -f "${0}")
echo " productionID: ${productionID}"
echo " batchCommand: ${batchCommand}"
echo " batchFlags: ${batchFlags}"
- echo " alirootSource: ${alirootSource}"
- echo " ALICE_ROOT: ${ALICE_ROOT}"
- echo " ALIROOT_RELEASE: ${ALICE_RELEASE}"
+ echo " alirootEnv: ${alirootEnv}"
+ ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
+ ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
echo " inputList: ${inputList}"
echo " configPath: ${configPath}"
echo " commonOutputPath: ${commonOutputPath}"
###############################################################################
#run one chunk with valgrind:
- if [[ -n ${runValgrind} ]]; then
- [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
- [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
- submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}"
+ if [[ -n ${profilingCommand} ]]; then
+ [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
+ [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
+ submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
fi
################################################################################
echo "requested success rate is ${percentProcessedFilesToContinue}%"
echo "merging will start after ${nFilesToWaitFor} jobs are done"
- submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
+ submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
- submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
+ submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
LASTJOB=${JOBID1wait}
fi #end running CPass0
[[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
done
- submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}"
+ submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
LASTJOB=${JOBID2}
if [[ -n ${generateMC} ]]; then
- submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done"
+ submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
LASTJOB="mrl${JOBpostfix}"
fi
echo "requested success rate is ${percentProcessedFilesToContinue}%"
echo "merging will start after ${nFilesToWaitFor} jobs are done"
- submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
+ submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
################################################################################
## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
- submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
+ submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
LASTJOB=${JOBID4wait}
################################################################################
[[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
done
- submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}"
+ submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
LASTJOB=${JOBID5}
echo
fi
- ##############################
- if [ ${runESDfiltering} -eq 1 ]; then
- rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
- rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
- echo
- echo submitting filtering for run ${runNumber}
- echo
- submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done "
- submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
- LASTJOB=${JOBfilterESDcpass1}
- fi
+ ###############################
+ #if [ ${runESDfiltering} -eq 1 ]; then
+ # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
+ # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
+ # echo
+ # echo submitting filtering for run ${runNumber}
+ # echo
+ # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
+ # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
+ # LASTJOB=${JOBfilterESDcpass1}
+ #fi
done
#################################################################################
#################################################################################
- if [ ${runESDfiltering} -eq 1 ]; then
- submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
- else
- submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
- fi
+ #if [ ${runESDfiltering} -eq 1 ]; then
+ # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
+ #else
+ submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
+ #fi
LASTJOB=${JOBID5wait}
- [[ -z ${runMakeSummary} ]] && runMakeSummary=0
- if [ ${runMakeSummary} -eq 1 ]; then
- echo
- echo "submit make a summary"
- echo
+ #################################################################################
+ echo
+ echo "submit make a summary"
+ echo
- submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}"
- LASTJOB=${JOBID6}
- fi
+ submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
+ LASTJOB=${JOBID6}
#################################################################################
#restore stdout
exec 1>&7 7>&-
echo "jobs submitted."
+ return 0
}
goWaitForOutput()
sleep 60
done
echo "DONE! exiting..."
+ return 0
}
mergeSysLogs()
awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
(( i++ ))
done < <(ls -1 ${inputFiles}) > ${outputFile}
+ return 0
}
goMakeMergedSummaryTree()
# summary logs
# qa plot making
# final file lists
+ #some defaults:
+ log="summary.log"
+ productionID="qa"
configFile=${1}
shift 1
- extraOpts="$@"
- log="summary.log"
- productionID="qa"
- parseConfig ${configFile} ${extraOpts}
+ extraOpts=("$@")
+ if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
[[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
rm -f trending.list
goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
rm -f filtering.list
- goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null
+ goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
#/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
rm -f cpass0.dcsTree.list
goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
#merge trending
rm -f ${commonOutputPath}/trending_merged.root
- goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log
+ goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
goMakeSummaryTree ${commonOutputPath} 0
goMakeSummaryTree ${commonOutputPath} 1
- goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log
+ goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log
#make a merged summary tree out of the QA trending, dcs trees and log summary trees
goMakeMergedSummaryTree
#if set, email the summary
- [[ -n ${mailSummaryTo} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${mailSummaryTo}
+ [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
return 0
}
echo >> ${outfile}
done
+ return 0
}
parseConfig()
{
configFile=${1}
shift
+ args=("$@")
+
+ #some defaults
+ #autoOCDB=0
+ defaultOCDB="raw://"
+ #runNumber=167123
+ #makeflowPath="/hera/alice/aux/cctools/bin"
+ #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
+ #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
+ makeflowOptions=""
+ batchCommand="/usr/bin/qsub"
+ batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
+ baseOutputDirectory="$PWD/output"
+ #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
+ #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
+ #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
+ reconstructInTemporaryDir=0
+ recoTriggerOptions="\"\""
+ percentProcessedFilesToContinue=100
+ maxSecondsToWait=$(( 3600*24 ))
+ nEvents=-1
+ nMaxChunks=0
+ postSetUpActionCPass0=""
+ postSetUpActionCPass1=""
+ runCPass0reco=1
+ runCPass0MergeMakeOCDB=1
+ runCPass1reco=1
+ runCPass1MergeMakeOCDB=1
+ runESDfiltering=1
+ filteringFactorHighPt=1e2
+ filteringFactorV0s=1e1
+ MAILTO=""
+ #pretend=1
+ #dontRedirectStdOutToLog=1
+ logToFinalDestination=1
+ ALIROOT_FORCE_COREDUMP=1
#first, source the config file
if [ -f ${configFile} ]; then
echo "config file ${configFile} not found!, skipping..."
fi
- #then, parse the options as theya override the optionf from file
- while [[ -n ${1} ]]; do
- var="${1#--}"
- echo "extraOpts: exporting ${var}"
- eval "export ${var}"
- shift
+ #then, parse the options as they override the options from file
+ for opt in "${args[@]}"; do
+ if [[ ! "${opt}" =~ .*=.* ]]; then
+ echo "badly formatted option ${var}, should be: option=value, stopping..."
+ return 1
+ fi
+ local var="${opt%%=*}"
+ local value="${opt#*=}"
+ echo "${var} = ${value}"
+ export ${var}="${value}"
done
+ #do some checking
+ [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
+
#export the aliroot function if defined to override normal behaviour
[[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
+
+ return 0
}
aliroot()
{
args="$@"
- if [[ -n ${valgrindCommand} ]]; then
+ if [[ -n ${useProfilingCommand} ]]; then
valgrindLogFile="cpu.txt"
[[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
[[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
- [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
- echo running ${valgrindCommand} aliroot ${args}
- ${valgrindCommand} aliroot ${args}
+ [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
+ echo running ${useProfilingCommand} aliroot ${args}
+ ${useProfilingCommand} aliroot ${args}
else
#to prevent an infinite recursion use "command aliroot" to disable
#aliases and functions
echo running command aliroot ${args}
- command aliroot ${args}
+ command aliroot "$@"
fi
+ return 0
}
guessRunData()
#ALL OK
return 0
fi
+ return 0
}
main "$@"
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff