| fc7e1e45 |
1 | #!/bin/bash |
| 71353a5f |
2 | #include benchmark.config |
| 3 | |
| fc7e1e45 |
4 | # this script runs the CPass0/CPass1 train |
| 5 | # produced OCDB updates are local |
| 6 | |
| 7 | main() |
| 8 | { |
| 9 | #run in proper mode depending on the selection |
| 71353a5f |
10 | if [[ $# -lt 1 ]]; then |
| 11 | if [[ ! "${0}" =~ "bash" ]]; then |
| 12 | echo "uses makeflow:" |
| 13 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" |
| 14 | echo "uses a batch system (SGE):" |
| 15 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" |
| 16 | echo "extraOpts if non-empty override the config file, e.g.:" |
| 17 | echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10" |
| 18 | fi |
| 19 | return |
| 20 | fi |
| 21 | |
| 22 | #define some aliases - default is to call one of the functions directly |
| 23 | runMode=${1} |
| fc7e1e45 |
24 | umask 0002 |
| 25 | shift |
| 71353a5f |
26 | case ${runMode} in |
| fc7e1e45 |
27 | "CPass0") goCPass0 "$@";; |
| 28 | "CPass1") goCPass1 "$@";; |
| 71353a5f |
29 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; |
| fc7e1e45 |
30 | "MergeCPass0") goMergeCPass0 "$@";; |
| 31 | "MergeCPass1") goMergeCPass1 "$@";; |
| cb473870 |
32 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; |
| 71353a5f |
33 | "MakeSummary") goMakeSummary "$@";; |
| cb473870 |
34 | "run") goSubmitMakeflow "$@";; |
| 35 | "submit") goSubmitBatch "$@";; |
| fc7e1e45 |
36 | "test") goTest "$@";; |
| 71353a5f |
37 | "GenerateMakeflow") goGenerateMakeflow "$@";; |
| 38 | "PrintValues") goPrintValues "$@";; |
| cb473870 |
39 | "CreateQAplots") goCreateQAplots "$@";; |
| 40 | "WaitForOutput") goWaitForOutput "$@";; |
| 71353a5f |
41 | "Merge") goMerge "$@";; |
| 42 | *) |
| 43 | eval ${runMode} $@ |
| fc7e1e45 |
44 | ;; |
| 45 | esac |
| 46 | } |
| 47 | |
| 71353a5f |
48 | generateMC() |
| 49 | { |
| 50 | #generate one raw chunk in current directory |
| 51 | SEED=${JOB_ID}${SGE_TASK_ID} |
| 52 | export CONFIG_SEED=${SEED} |
| 53 | runNumber=${1} |
| 54 | OCDBpath=${2} |
| 55 | nEventsim=${3} |
| 56 | if [[ -n ${pretend} ]]; then |
| 57 | touch galice.root |
| 58 | else |
| 59 | if [[ -f sim.C && -f Config.C ]] ; then |
| 60 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 |
| 61 | mv syswatch.log simwatch.log |
| 62 | fi |
| 63 | fi |
| 64 | } |
| 65 | |
| fc7e1e45 |
66 | goCPass0() |
| 67 | { |
| 68 | umask 0002 |
| 69 | |
| 71353a5f |
70 | targetDirectory=${1} |
| 71 | inputList=${2} |
| 72 | nEvents=${3} |
| 73 | ocdbPath=${4} |
| 74 | configFile=${5} |
| 75 | runNumber=${6} |
| 76 | jobindex=${7} |
| 77 | shift 7 |
| 78 | parseConfig ${configFile} $@ |
| 79 | |
| 80 | #use the jobindex only if set and non-negative |
| 81 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 82 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 83 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 84 | fi |
| 85 | |
| 86 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 87 | doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" |
| 88 | |
| 89 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 90 | |
| 91 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 92 | ulimit -c unlimited |
| 93 | export ALIROOT_FORCE_COREDUMP |
| 94 | fi |
| fc7e1e45 |
95 | |
| cb473870 |
96 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 97 | #at the QA stage the pass number is guessed from the path stored here. |
| 98 | #The Format is: |
| 99 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 100 | export PRODUCTION_METADATA="OutputDir=cpass0" |
| 101 | |
| 71353a5f |
102 | #runCPassX/C expects the raw chunk to be linked in the run dir |
| 103 | #despite it being accessed by the full path |
| 104 | chunkName=${infile##*/} |
| 105 | ln -s ${infile} ${runpath}/${chunkName} |
| 106 | |
| 107 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 108 | mkdir -p ${outputDir} |
| 109 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 |
| 110 | |
| 111 | #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} |
| 112 | runpath=${outputDir} |
| 113 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 114 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 115 | |
| 116 | mkdir -p ${runpath} |
| 117 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 |
| 118 | cd ${runpath} |
| fc7e1e45 |
119 | |
| fc7e1e45 |
120 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 71353a5f |
121 | infile=${inputList} |
| fc7e1e45 |
122 | else |
| 71353a5f |
123 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| fc7e1e45 |
124 | fi |
| fc7e1e45 |
125 | |
| 71353a5f |
126 | #####MC |
| 127 | if [[ -n ${generateMC} ]]; then |
| 128 | olddir=${PWD} |
| 129 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} |
| 130 | simrunpath=${outputDirMC} |
| 131 | [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} |
| 132 | [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d) |
| 133 | mkdir -p ${outputDirMC} |
| 134 | mkdir -p ${simrunpath} |
| 135 | cd ${simrunpath} |
| 136 | |
| 137 | filesMC=( |
| 138 | "${commonOutputPath}/sim.C" |
| 139 | "${commonOutputPath}/rec.C" |
| 140 | "${commonOutputPath}/Config.C" |
| 141 | "${commonOutputPath}/OCDB_*.root" |
| 142 | ) |
| 143 | for file in ${filesMC[*]}; do |
| 144 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 145 | done |
| fc7e1e45 |
146 | |
| 71353a5f |
147 | generateMC ${runNumber} ${ocdbPath} ${nEvents} |
| fc7e1e45 |
148 | |
| 71353a5f |
149 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works |
| 150 | cd ${olddir} |
| 151 | |
| 152 | ln -s ${outputDirMC}/* ${runpath}/ |
| 153 | |
| 154 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! |
| 155 | infile="" |
| 156 | fi |
| 157 | ###### |
| 158 | |
| 159 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 |
| 160 | |
| 161 | logOutputDir=${runpath} |
| 162 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 163 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout |
| 164 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 165 | echo "${0} $*" |
| fc7e1e45 |
166 | |
| 167 | echo "#####################" |
| 168 | echo CPass0: |
| 169 | echo JOB setup |
| 71353a5f |
170 | echo nEvents ${nEvents} |
| 171 | echo runNumber ${runNumber} |
| 172 | echo ocdbPath ${ocdbPath} |
| 173 | echo infile ${infile} |
| 174 | echo chunkName ${chunkName} |
| 175 | echo jobindex ${jobindex} |
| 176 | echo recoTriggerOptions ${recoTriggerOptions} |
| 177 | echo targetDirectory ${targetDirectory} |
| 178 | echo commonOutputPath ${commonOutputPath} |
| 179 | echo doneFile ${doneFile} |
| 180 | echo runpath ${runpath} |
| 181 | echo outputDir ${outputDir} |
| 182 | echo ALICE_ROOT ${ALICE_ROOT} |
| 183 | echo PWD ${PWD} |
| fc7e1e45 |
184 | echo "########## ###########" |
| 185 | |
| 71353a5f |
186 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
187 | |
| 188 | filesCPass0=( |
| 71353a5f |
189 | "${commonOutputPath}/runCPass0.sh" |
| 190 | "${commonOutputPath}/recCPass0.C" |
| 191 | "${commonOutputPath}/runCalibTrain.C" |
| 192 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 193 | "${commonOutputPath}/OCDB.root" |
| 194 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
| 195 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" |
| 196 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" |
| fc7e1e45 |
197 | ) |
| 198 | |
| 199 | for file in ${filesCPass0[*]}; do |
| 200 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 201 | done |
| 202 | |
| 71353a5f |
203 | echo "this directory (${PWD}) contents:" |
| 204 | /bin/ls |
| fc7e1e45 |
205 | echo |
| 206 | chmod u+x runCPass0.sh |
| 207 | |
| 71353a5f |
208 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 209 | |
| 210 | if [[ -n ${postSetUpActionCPass0} ]]; then |
| 211 | echo "running ${postSetUpActionCPass0}" |
| 212 | eval ${postSetUpActionCPass0} |
| fc7e1e45 |
213 | fi |
| 214 | |
| 215 | #run CPass0 |
| 71353a5f |
216 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 217 | if [[ -n ${pretend} ]]; then |
| cb473870 |
218 | touch AliESDs.root |
| 219 | touch AliESDfriends.root |
| fc7e1e45 |
220 | touch AliESDfriends_v1.root |
| 221 | touch rec.log |
| 222 | touch calib.log |
| 223 | else |
| 71353a5f |
224 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| 225 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| fc7e1e45 |
226 | fi |
| 227 | |
| fc7e1e45 |
228 | #move stuff to final destination |
| 71353a5f |
229 | echo "this directory (${PWD}) contents:" |
| 230 | /bin/ls |
| fc7e1e45 |
231 | echo |
| 232 | |
| 71353a5f |
233 | echo rm -f ./${chunkName} |
| 234 | rm -f ./${chunkName} |
| 235 | echo "cp --recursive ${runpath}/* ${outputDir}" |
| 236 | cp -p --recursive ${runpath}/* ${outputDir} |
| fc7e1e45 |
237 | echo |
| 5ed7c82c |
238 | |
| 239 | #validate CPass0 |
| 240 | cd ${outputDir} |
| 71353a5f |
241 | touch ${doneFile} |
| 242 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile} |
| 243 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} |
| 244 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile} |
| 245 | echo "dir ${outputDir}" >> ${doneFile} |
| 246 | summarizeLogs >> ${doneFile} |
| fc7e1e45 |
247 | |
| 71353a5f |
248 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| fc7e1e45 |
249 | } |
| 250 | |
| 251 | goCPass1() |
| 252 | { |
| 253 | umask 0002 |
| 254 | |
| 71353a5f |
255 | targetDirectory=${1} |
| 256 | inputList=${2} |
| 257 | nEvents=${3} |
| 258 | ocdbPath=${4} |
| 259 | configFile=${5} |
| 260 | runNumber=${6} |
| 261 | jobindex=${7} |
| 262 | shift 7 |
| 263 | parseConfig ${configFile} $@ |
| 264 | |
| 265 | #use the jobindex only if set and non-negative |
| 266 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 267 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 268 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 269 | fi |
| 270 | |
| 271 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 272 | doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" |
| 273 | |
| 274 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 275 | |
| 276 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 277 | ulimit -c unlimited |
| 278 | export ALIROOT_FORCE_COREDUMP |
| 279 | fi |
| fc7e1e45 |
280 | |
| cb473870 |
281 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 282 | #at the QA stage the pass number is guessed from the path stored here. |
| 283 | #The Format is: |
| 284 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 285 | export PRODUCTION_METADATA="OutputDir=cpass1" |
| 286 | |
| 71353a5f |
287 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 |
| fc7e1e45 |
288 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 71353a5f |
289 | infile=${inputList} |
| fc7e1e45 |
290 | else |
| 71353a5f |
291 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| fc7e1e45 |
292 | fi |
| 293 | |
| 294 | chunkName=${infile##*/} |
| 71353a5f |
295 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 296 | mkdir -p ${outputDir} |
| 297 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 |
| fc7e1e45 |
298 | |
| 71353a5f |
299 | #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} |
| 300 | runpath=${outputDir} |
| 301 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 302 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 303 | |
| 304 | #MC |
| 305 | if [[ "${infile}" =~ galice\.root ]]; then |
| 306 | ln -s ${inputList%/*}/* ${runpath} |
| 307 | infile="" |
| 308 | fi |
| fc7e1e45 |
309 | |
| 310 | #init the running path |
| 71353a5f |
311 | mkdir -p ${runpath} |
| 312 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 |
| 313 | cd ${runpath} |
| fc7e1e45 |
314 | |
| 71353a5f |
315 | #this is needed for runCPass1.sh |
| 316 | ln -s ${infile} ${runpath}/${chunkName} |
| fc7e1e45 |
317 | |
| 71353a5f |
318 | logOutputDir=${runpath} |
| 319 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 320 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout |
| 321 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 322 | echo "${0} $*" |
| fc7e1e45 |
323 | |
| 324 | echo "#####################" |
| 325 | echo CPass1: |
| 326 | echo JOB setup |
| 71353a5f |
327 | echo nEvents ${nEvents} |
| 328 | echo runNumber ${runNumber} |
| 329 | echo ocdbPath ${ocdbPath} |
| 330 | echo infile ${infile} |
| 331 | echo chunkName ${chunkName} |
| 332 | echo jobindex ${jobindex} |
| 333 | echo recoTriggerOptions ${recoTriggerOptions} |
| 334 | echo targetDirectory ${targetDirectory} |
| 335 | echo commonOutputPath ${commonOutputPath} |
| 336 | echo doneFile ${doneFile} |
| 337 | echo runpath ${runpath} |
| 338 | echo outputDir ${outputDir} |
| 339 | echo ALICE_ROOT ${ALICE_ROOT} |
| 340 | echo PWD ${PWD} |
| fc7e1e45 |
341 | echo "########## ###########" |
| 342 | |
| 71353a5f |
343 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
344 | |
| 345 | filesCPass1=( |
| 71353a5f |
346 | "${commonOutputPath}/runCPass1.sh" |
| 347 | "${commonOutputPath}/recCPass1.C" |
| 348 | "${commonOutputPath}/recCPass1_OuterDet.C" |
| 349 | "${commonOutputPath}/runCalibTrain.C" |
| 350 | "${commonOutputPath}/QAtrain_duo.C" |
| 351 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 352 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 353 | "${commonOutputPath}/OCDB.root" |
| 354 | "${trustedQAtrainMacro}" |
| 355 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
| 356 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" |
| 357 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" |
| 358 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" |
| 359 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
360 | ) |
| 361 | |
| 362 | for file in ${filesCPass1[*]}; do |
| 363 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 364 | done |
| 365 | |
| 71353a5f |
366 | echo "this directory (${PWD}) contents:" |
| 367 | /bin/ls |
| fc7e1e45 |
368 | echo |
| 369 | |
| 71353a5f |
370 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 371 | |
| 372 | if [[ -n ${postSetUpActionCPass1} ]]; then |
| 373 | echo "running ${postSetUpActionCPass1}" |
| 374 | eval ${postSetUpActionCPass1} |
| fc7e1e45 |
375 | echo |
| 376 | fi |
| 377 | |
| 378 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 379 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
380 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 381 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
382 | else |
| 383 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 384 | fi |
| 385 | |
| 71353a5f |
386 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then |
| cb473870 |
387 | echo "cpass0 produced no calibration! exiting..." |
| 71353a5f |
388 | touch ${doneFile} |
| 389 | return 1 |
| cb473870 |
390 | fi |
| fc7e1e45 |
391 | |
| 71353a5f |
392 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory |
| 393 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB |
| 394 | #entries using a relative path, e.g. local://./OCDB |
| 395 | mkdir Barrel OuterDet |
| 396 | ln -s ../OCDB Barrel/OCDB |
| 397 | ln -s ../OCDB OuterDet/OCDB |
| 398 | |
| fc7e1e45 |
399 | #run CPass1 |
| 400 | chmod u+x runCPass1.sh |
| 71353a5f |
401 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 402 | if [[ -n ${pretend} ]]; then |
| cb473870 |
403 | touch AliESDs_Barrel.root |
| 404 | touch AliESDfriends_Barrel.root |
| fc7e1e45 |
405 | touch AliESDfriends_v1.root |
| cb473870 |
406 | touch QAresults_barrel.root |
| 407 | touch EventStat_temp_barrel.root |
| 408 | touch AODtpITS.root |
| 409 | touch AliESDs_Outer.root |
| 410 | touch AliESDfriends_Outer.root |
| 411 | touch QAresults_outer.root |
| 412 | touch EventStat_temp_outer.root |
| fc7e1e45 |
413 | touch rec.log |
| 414 | touch calib.log |
| 415 | touch qa.log |
| 416 | else |
| 71353a5f |
417 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| fc7e1e45 |
418 | fi |
| cb473870 |
419 | |
| 420 | ##handle possible crashes in QA (happens often in trunk) |
| 421 | ##rerun QA with a trusted aliroot version |
| 422 | #if [[ $(validateLog qa_barrel.log) ]]; then |
| 423 | # echo "qa_barrel.log not validated!" |
| 424 | #fi |
| 425 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then |
| 71353a5f |
426 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" |
| cb473870 |
427 | # source ${setupTrustedAliROOTenvInCurrentShell} |
| 428 | # cd Barrel |
| 429 | # rm QAresults_barrel.root |
| 430 | # rm EventStat_temp_barrel.root |
| 431 | # rm AODtpITS.root |
| 432 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root |
| 433 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root |
| 71353a5f |
434 | # if [[ -n ${trustedQAtrainMacro} ]]; then |
| 435 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" |
| cb473870 |
436 | # fi |
| 71353a5f |
437 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" |
| 438 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log |
| cb473870 |
439 | # cd ../ |
| 440 | #fi |
| 441 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root |
| 442 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root |
| 443 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . |
| 444 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . |
| 445 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root |
| 446 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root |
| 447 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . |
| 448 | |
| fc7e1e45 |
449 | #move stuff to final destination |
| 71353a5f |
450 | echo "this directory (${PWD}) contents:" |
| 451 | /bin/ls |
| 452 | echo rm -f ./${chunkName} |
| 453 | rm -f ./${chunkName} |
| fc7e1e45 |
454 | echo "cp --recursive ${runpath}/* ${outputDir}" |
| 71353a5f |
455 | cp -pf --recursive ${runpath}/* ${outputDir} |
| fc7e1e45 |
456 | echo |
| 457 | |
| 5ed7c82c |
458 | #validate CPass1 |
| 459 | cd ${outputDir} |
| 71353a5f |
460 | touch ${doneFile} |
| 461 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile} |
| 462 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} |
| 463 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile} |
| 464 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile} |
| 465 | echo "dir ${outputDir}" >> ${doneFile} |
| 466 | summarizeLogs >> ${doneFile} |
| 5ed7c82c |
467 | |
| 71353a5f |
468 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| fc7e1e45 |
469 | } |
| 470 | |
| 471 | |
| 472 | goMergeCPass0() |
| 473 | { |
| fc7e1e45 |
474 | # |
| 475 | # find the output files and merge them |
| 476 | # |
| 477 | |
| 71353a5f |
478 | outputDir=${1} |
| 479 | ocdbStorage=${2} |
| 480 | configFile=${3} |
| 481 | runNumber=${4} |
| 482 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly |
| 483 | shift 5 |
| 484 | parseConfig ${configFile} $@ |
| 485 | |
| 486 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 487 | doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" |
| 488 | |
| 489 | umask 0002 |
| 490 | ulimit -c unlimited |
| fc7e1e45 |
491 | |
| 71353a5f |
492 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
493 | |
| 71353a5f |
494 | #runpath=${PWD}/rundir_cpass0_Merge_${runNumber} |
| 495 | runpath=${outputDir} |
| 496 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 497 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| fc7e1e45 |
498 | |
| 71353a5f |
499 | mkdir -p ${runpath} |
| 500 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 |
| 501 | cd ${runpath} |
| fc7e1e45 |
502 | |
| 71353a5f |
503 | logOutputDir=${runpath} |
| 504 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 505 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log |
| 506 | echo "${0} $*" |
| fc7e1e45 |
507 | |
| fc7e1e45 |
508 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| fc7e1e45 |
509 | |
| 510 | echo goMergeCPass0 SETUP: |
| 71353a5f |
511 | echo runNumber=${runNumber} |
| 512 | echo outputDir=${outputDir} |
| 513 | echo ocdbStorage=${ocdbStorage} |
| 514 | echo calibrationFilesToMerge=${calibrationFilesToMerge} |
| 515 | echo mergingScript=${mergingScript} |
| 516 | echo commonOutputPath=${commonOutputPath} |
| 517 | echo runpath=${runpath} |
| fc7e1e45 |
518 | |
| 519 | # copy files in case they are not already there |
| 520 | filesMergeCPass0=( |
| 71353a5f |
521 | "${commonOutputPath}/${calibrationFilesToMerge}" |
| 522 | "${commonOutputPath}/OCDB.root" |
| 523 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 524 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
| 525 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" |
| 526 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" |
| 527 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" |
| 528 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" |
| fc7e1e45 |
529 | ) |
| 530 | for file in ${filesMergeCPass0[*]}; do |
| 531 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 532 | done |
| 533 | |
| 71353a5f |
534 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 535 | |
| 536 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
537 | |
| 538 | # |
| cb473870 |
539 | echo "PWD" |
| 71353a5f |
540 | /bin/ls |
| cb473870 |
541 | echo "PWD/.." |
| 71353a5f |
542 | /bin/ls ../ |
| cb473870 |
543 | |
| fc7e1e45 |
544 | |
| 545 | #merge calibration |
| 71353a5f |
546 | chmod u+x ${mergingScript} |
| fc7e1e45 |
547 | mkdir -p ./OCDB |
| 71353a5f |
548 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 549 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" |
| 550 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} |
| fc7e1e45 |
551 | fi |
| 552 | |
| 71353a5f |
553 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 554 | if [[ -n ${pretend} ]]; then |
| fc7e1e45 |
555 | touch CalibObjects.root |
| 556 | touch ocdb.log |
| 557 | touch merge.log |
| 71353a5f |
558 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ |
| 559 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ |
| 560 | touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root |
| 561 | touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root |
| fc7e1e45 |
562 | else |
| 71353a5f |
563 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
| fc7e1e45 |
564 | fi |
| 565 | |
| 566 | ### produce the output |
| 567 | #tar the produced OCDB for reuse |
| 71353a5f |
568 | tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB |
| fc7e1e45 |
569 | |
| 71353a5f |
570 | #produce the calib trees for expert QA (dcsTime.root) |
| 571 | goMakeLocalOCDBaccessConfig ./OCDB |
| 572 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 573 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 574 | |
| 575 | /bin/ls |
| 5ed7c82c |
576 | |
| 577 | #copy all to output dir |
| 71353a5f |
578 | cp -pf --recursive ${runpath}/* ${outputDir} |
| 5ed7c82c |
579 | |
| 71353a5f |
580 | if [[ -n ${generateMC} ]]; then |
| 581 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done |
| 582 | fi |
| 583 | |
| fc7e1e45 |
584 | #validate merging cpass0 |
| 5ed7c82c |
585 | cd ${outputDir} |
| 71353a5f |
586 | touch ${doneFile} |
| 587 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} |
| 588 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} |
| 589 | echo "dir ${outputDir}" >> ${doneFile} |
| 590 | summarizeLogs >> ${doneFile} |
| fc7e1e45 |
591 | |
| 71353a5f |
592 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| fc7e1e45 |
593 | } |
| 594 | |
| 595 | goMergeCPass1() |
| 596 | { |
| fc7e1e45 |
597 | # |
| 598 | # find the output files and merge them |
| 599 | # |
| 600 | |
| 71353a5f |
601 | outputDir=${1} |
| 602 | ocdbStorage=${2} |
| 603 | configFile=${3} |
| 604 | runNumber=${4} |
| 605 | calibrationFilesToMerge=${5} |
| 606 | qaFilesToMerge=${6} |
| 607 | shift 6 |
| 608 | parseConfig ${configFile} $@ |
| 609 | |
| 610 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 611 | doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" |
| 612 | |
| 613 | umask 0002 |
| 614 | ulimit -c unlimited |
| fc7e1e45 |
615 | |
| 616 | #clean up first: |
| 71353a5f |
617 | rm -f ${outputDir}/*.log |
| 618 | rm -f ${outputDir}/*.root |
| 619 | rm -f ${outputDir}/*done |
| 620 | |
| 621 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 622 | |
| 623 | #runpath=${PWD}/rundir_cpass1_Merge_${runNumber} |
| 624 | runpath=${outputDir} |
| 625 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 626 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 627 | |
| 628 | mkdir -p ${runpath} |
| 629 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 |
| 630 | cd ${runpath} |
| 631 | |
| 632 | logOutputDir=${runpath} |
| 633 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 634 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log |
| 635 | echo "${0} $*" |
| 636 | |
| 637 | calibrationOutputFileName='AliESDfriends_v1.root' |
| 638 | qaOutputFileName='QAresults*.root' |
| fc7e1e45 |
639 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| cb473870 |
640 | #important to have the string "Stage.txt" in the filename to trigger the merging |
| 641 | #it has to be a list of directories containing the files |
| fc7e1e45 |
642 | qaMergedOutputFileName="QAresults_merged.root" |
| 643 | |
| 644 | echo goMergeCPass1 SETUP: |
| 71353a5f |
645 | echo runNumber=${runNumber} |
| 646 | echo outputDir=${outputDir} |
| 647 | echo ocdbStorage=${ocdbStorage} |
| 648 | echo calibrationFilesToMerge=$calibrationFilesToMerge |
| 649 | echo qaFilesToMerge=$qaFilesToMerge |
| 650 | echo calibrationOutputFileName=${calibrationOutputFileName} |
| 651 | echo mergingScript=${mergingScript} |
| fc7e1e45 |
652 | |
| 653 | # copy files in case they are not already there |
| 654 | filesMergeCPass1=( |
| 71353a5f |
655 | "${commonOutputPath}/${calibrationFilesToMerge}" |
| 656 | "${commonOutputPath}/${qaFilesToMerge}" |
| 657 | "${commonOutputPath}/OCDB.root" |
| 658 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 659 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 660 | "${commonOutputPath}/QAtrain_duo.C" |
| 661 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
| 662 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" |
| 663 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" |
| 664 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" |
| 665 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" |
| 666 | "${trustedQAtrainMacro}" |
| 667 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
668 | ) |
| 669 | for file in ${filesMergeCPass1[*]}; do |
| 670 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 671 | done |
| 672 | |
| 71353a5f |
673 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 674 | |
| cb473870 |
675 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 676 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
677 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 678 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
679 | else |
| 680 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 681 | fi |
| 682 | |
| 71353a5f |
683 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
684 | |
| 685 | # |
| 71353a5f |
686 | /bin/ls |
| fc7e1e45 |
687 | |
| 688 | #merge calibration |
| 71353a5f |
689 | chmod u+x ${mergingScript} |
| fc7e1e45 |
690 | mkdir -p OCDB |
| 71353a5f |
691 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 692 | echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}" |
| 693 | /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge} |
| fc7e1e45 |
694 | fi |
| 695 | |
| 71353a5f |
696 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 697 | if [[ -n ${pretend} ]]; then |
| fc7e1e45 |
698 | touch CalibObjects.root |
| 699 | touch ocdb.log |
| 700 | touch merge.log |
| 71353a5f |
701 | touch cpass1.localOCDB.${runNumber}.tgz |
| fc7e1e45 |
702 | else |
| 71353a5f |
703 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
| fc7e1e45 |
704 | fi |
| 705 | |
| 71353a5f |
706 | tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB |
| fc7e1e45 |
707 | |
| 708 | #merge QA |
| 5ed7c82c |
709 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF |
| 710 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF |
| 711 | |
| 71353a5f |
712 | if [[ ! -f ${qaFilesToMerge} ]]; then |
| cb473870 |
713 | #find the files, but only store the directories (QAtrain_duo.C requires this) |
| 71353a5f |
714 | echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" |
| 715 | eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} |
| fc7e1e45 |
716 | fi |
| 717 | |
| cb473870 |
718 | #do the merge |
| 71353a5f |
719 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 720 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" |
| 721 | if [[ -n ${pretend} ]]; then |
| 722 | touch ${qaMergedOutputFileName} |
| fc7e1e45 |
723 | touch merge.log |
| cb473870 |
724 | touch trending.root |
| fc7e1e45 |
725 | else |
| 71353a5f |
726 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 727 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log |
| 728 | mv QAresults_barrel.root ${qaMergedOutputFileName} |
| cb473870 |
729 | mv trending_barrel.root trending.root |
| fc7e1e45 |
730 | fi |
| cb473870 |
731 | |
| 71353a5f |
732 | #produce the calib trees for expert QA |
| 733 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 734 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 735 | |
| 736 | /bin/ls |
| 5ed7c82c |
737 | |
| 738 | #copy all to output dir |
| 71353a5f |
739 | cp -pf --recursive ${runpath}/* ${outputDir} |
| 5ed7c82c |
740 | |
| fc7e1e45 |
741 | #validate merge cpass1 |
| 5ed7c82c |
742 | cd ${outputDir} |
| 71353a5f |
743 | touch ${doneFile} |
| 744 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} |
| 745 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile} |
| 746 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile} |
| 747 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} |
| 748 | echo "dir ${outputDir}" >> ${doneFile} |
| 749 | summarizeLogs >> ${doneFile} |
| 750 | |
| 751 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| cb473870 |
752 | } |
| 753 | |
| 754 | goMerge() |
| 755 | { |
| 756 | #generic root merge using CPass1 merge.C script |
| 71353a5f |
757 | inputList=${1} |
| 758 | outputFile=${2} |
| cb473870 |
759 | configFile=${3-"becnhmark.config"} |
| 71353a5f |
760 | shift 3 |
| 761 | parseConfig ${configFile} $@ |
| fc7e1e45 |
762 | |
| 71353a5f |
763 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 |
| 764 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 |
| 765 | umask 0002 |
| 766 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 767 | rm -f ${outputFile} |
| 768 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log |
| 5dfac970 |
769 | } |
| fc7e1e45 |
770 | |
| cb473870 |
771 | goSubmitMakeflow() |
| 5dfac970 |
772 | { |
| cb473870 |
773 | #run |
| 71353a5f |
774 | productionID=${1} |
| 775 | inputFileList=${2} |
| 776 | configFile=${3} |
| 777 | shift 3 |
| 778 | extraOpts="$@" |
| 779 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
780 | |
| 5dfac970 |
781 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 782 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
783 | |
| 71353a5f |
784 | #create the directopry for the metadata |
| 785 | mkdir meta |
| 786 | |
| 787 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then |
| cb473870 |
788 | echo "setting the makflow path from the config: " |
| 789 | echo " export PATH=${makeflowPath}:${PATH}" |
| 790 | export PATH=${makeflowPath}:${PATH} |
| 791 | fi |
| 792 | |
| 793 | #submit - use makeflow if available, fall back to old stuff when makeflow not there |
| 794 | if which makeflow; then |
| cb473870 |
795 | |
| 71353a5f |
796 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow |
| 797 | |
| cb473870 |
798 | makeflow ${makeflowOptions} benchmark.makeflow |
| 799 | cd ../ |
| 800 | else |
| 801 | echo "no makeflow!" |
| 802 | fi |
| 5dfac970 |
803 | } |
| fc7e1e45 |
804 | |
| 5dfac970 |
805 | goGenerateMakeflow() |
| 806 | { |
| 807 | #generate the makeflow file |
| 71353a5f |
808 | productionID=${1} |
| 809 | inputFileList=${2} |
| 810 | configFile=${3} |
| 811 | shift 3 |
| 812 | extraOpts="$@" |
| 813 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
814 | |
| 5dfac970 |
815 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 816 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
817 | |
| 5dfac970 |
818 | commonOutputPath=${baseOutputDirectory}/${productionID} |
| fc7e1e45 |
819 | |
| 5dfac970 |
820 | #these files will be made a dependency - will be copied to the working dir of the jobs |
| 821 | declare -a copyFiles |
| 822 | inputFiles=( |
| 823 | "OCDB.root" |
| 824 | "localOCDBaccessConfig.C" |
| cb473870 |
825 | "QAtrain_duo.C" |
| 826 | "runCPass1.sh" |
| 827 | "recCPass1.C" |
| 828 | "recCPass1_OuterDet.C" |
| 829 | "runCalibTrain.C" |
| 830 | "runCPass0.sh" |
| 831 | "recCPass0.C" |
| 5dfac970 |
832 | ) |
| 833 | for file in ${inputFiles[*]}; do |
| 834 | [[ -f ${file} ]] && copyFiles+=("${file}") |
| 835 | done |
| 836 | |
| 837 | #create the makeflow file |
| cb473870 |
838 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" |
| 5dfac970 |
839 | declare -a arr_cpass1_final |
| 71353a5f |
840 | listOfRuns[0]=${runNumber} |
| 841 | [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq)) |
| 5dfac970 |
842 | runindex=0 |
| fc7e1e45 |
843 | for runNumber in ${listOfRuns[*]}; do |
| 71353a5f |
844 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
845 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| fc7e1e45 |
846 | |
| cb473870 |
847 | unset arr_cpass0_outputs_onerun |
| 848 | unset arr_cpass1_outputs_onerun |
| 849 | declare -a arr_cpass0_outputs_onerun |
| 850 | declare -a arr_cpass1_outputs_onerun |
| fc7e1e45 |
851 | |
| cb473870 |
852 | jobindex=0 |
| 5dfac970 |
853 | while read inputFile; do |
| 854 | currentDefaultOCDB=${defaultOCDB} |
| cb473870 |
855 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 856 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
857 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) |
| cb473870 |
858 | fi |
| fc7e1e45 |
859 | |
| 5dfac970 |
860 | #CPass0 |
| 71353a5f |
861 | arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done" |
| 862 | echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}" |
| 863 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}" |
| fc7e1e45 |
864 | echo |
| 5dfac970 |
865 | |
| 866 | #CPass1 |
| 71353a5f |
867 | arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done" |
| 868 | echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" |
| 869 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}" |
| fc7e1e45 |
870 | echo |
| 5dfac970 |
871 | ((jobindex++)) |
| fc7e1e45 |
872 | |
| 71353a5f |
873 | done< <(grep "/000${runNumber}/" ${inputFileList}) |
| cb473870 |
874 | |
| 5dfac970 |
875 | #CPass0 list of Calib files to merge |
| 71353a5f |
876 | arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list" |
| 877 | echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}" |
| 878 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}" |
| 879 | echo |
| 880 | |
| 881 | #CPass0 merging |
| 882 | arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done" |
| 883 | echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}" |
| 884 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}" |
| 5dfac970 |
885 | echo |
| fc7e1e45 |
886 | |
| 5dfac970 |
887 | #CPass1 list of Calib/QA files to merge |
| cb473870 |
888 | # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger |
| 8d6a0a43 |
889 | # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid |
| 890 | # this corresponds to the last merging stage) |
| 71353a5f |
891 | arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list" |
| 892 | echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 893 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};" |
| cb473870 |
894 | echo |
| 71353a5f |
895 | arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" |
| 896 | echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 897 | echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}" |
| 5dfac970 |
898 | echo |
| fc7e1e45 |
899 | |
| 71353a5f |
900 | #CPass1 merging |
| 901 | arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done" |
| 902 | echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}" |
| 903 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}" |
| 5dfac970 |
904 | echo |
| fc7e1e45 |
905 | |
| 71353a5f |
906 | arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list" |
| 907 | echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 908 | echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}" |
| cb473870 |
909 | |
| 71353a5f |
910 | if [ ${runESDfiltering} -eq 1 ]; then |
| 911 | arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done" |
| 912 | echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}" |
| 913 | echo " ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}" |
| cb473870 |
914 | fi |
| 915 | |
| 5dfac970 |
916 | ((runindex++)) |
| 917 | done |
| fc7e1e45 |
918 | |
| 5dfac970 |
919 | #Summary |
| 71353a5f |
920 | echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}" |
| 921 | echo " LOCAL ./benchmark.sh MakeSummary ${configFile}" |
| cb473870 |
922 | echo |
| 923 | |
| 924 | } |
| 925 | |
| 926 | goPrintValues() |
| 927 | { |
| 928 | #print the values given the key from any number of files (store in output file on second argument) |
| 71353a5f |
929 | if [[ $# -lt 3 ]]; then |
| 930 | echo "goPrintValues key outputFile inputFiles" |
| 931 | echo "if outputFile is \"-\" dont redirect to a file" |
| 932 | return |
| 933 | fi |
| 934 | key=${1} |
| 935 | outputFile=${2} |
| 936 | [[ ${outputFile} =~ "-" ]] && outputFile="" |
| cb473870 |
937 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk |
| 71353a5f |
938 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} |
| 5dfac970 |
939 | } |
| fc7e1e45 |
940 | |
| 5dfac970 |
941 | goCreateQAplots() |
| 942 | { |
| 943 | umask 0002 |
| 71353a5f |
944 | mergedQAfileList=${1} |
| 945 | productionID=${2} |
| 946 | outputDir=${3} |
| 947 | configFile=${4} |
| 948 | shift 4 |
| 949 | parseConfig ${configFile} ${@} |
| 950 | |
| 951 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
952 | |
| 71353a5f |
953 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} |
| 954 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log |
| 955 | echo "${0} $*" |
| 956 | |
| 957 | olddir=${PWD} |
| 958 | mkdir -p ${outputDir} |
| 959 | cd ${outputDir} |
| 960 | [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1 |
| fc7e1e45 |
961 | |
| 71353a5f |
962 | echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList} |
| 963 | ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList} |
| fc7e1e45 |
964 | |
| 71353a5f |
965 | cd ${olddir} |
| 5dfac970 |
966 | } |
| fc7e1e45 |
967 | |
| 5dfac970 |
968 | goTest() |
| 969 | { |
| 970 | umask 0002 |
| 971 | exec 2>&1 |
| 972 | exec > >(tee test.log) |
| 973 | echo "$@" |
| 974 | echo something |
| 975 | } |
| fc7e1e45 |
976 | |
| 5dfac970 |
977 | alirootInfo() |
| 978 | { |
| 5dfac970 |
979 | umask 0002 |
| 71353a5f |
980 | # save aliroot repository info |
| 981 | [[ -z "${ALICE_ROOT}" ]] && return 1 |
| 982 | |
| 983 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" |
| 984 | echo "\${ROOTSYS}=${ROOTSYS}" |
| 985 | echo "\${PATH}=${PATH}" |
| 986 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" |
| 987 | |
| 988 | pushd ${PWD} |
| 989 | cd ${ALICE_ROOT} |
| 5dfac970 |
990 | echo |
| 71353a5f |
991 | |
| 992 | currentBranch=$(git rev-parse --abbrev-ref HEAD) |
| 993 | git status |
| 5dfac970 |
994 | echo "" |
| 995 | echo "" |
| 71353a5f |
996 | git diff ${currentBranch} |
| 997 | popd |
| 5dfac970 |
998 | } |
| fc7e1e45 |
999 | |
| 5dfac970 |
1000 | setYear() |
| 1001 | { |
| 1002 | #set the year |
| 71353a5f |
1003 | # ${1} - year to be set |
| 1004 | # ${2} - where to set the year |
| 1005 | year1=$(guessYear ${1}) |
| 1006 | year2=$(guessYear ${2}) |
| 1007 | local path=${2} |
| 1008 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} |
| 1009 | echo ${path} |
| 5dfac970 |
1010 | } |
| fc7e1e45 |
1011 | |
| 5dfac970 |
1012 | guessPeriod() |
| 1013 | { |
| 1014 | #guess the period from the path, pick the rightmost one |
| 1015 | local IFS="/" |
| 71353a5f |
1016 | declare -a path=( ${1} ) |
| 5dfac970 |
1017 | local dirDepth=${#path[*]} |
| 1018 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1019 | local field=${path[${x}]} |
| 71353a5f |
1020 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break |
| 5dfac970 |
1021 | done |
| 71353a5f |
1022 | echo ${period} |
| 5dfac970 |
1023 | } |
| fc7e1e45 |
1024 | |
| 5dfac970 |
1025 | guessYear() |
| 1026 | { |
| 1027 | #guess the year from the path, pick the rightmost one |
| 1028 | local IFS="/" |
| 71353a5f |
1029 | declare -a path=( ${1} ) |
| 5dfac970 |
1030 | local dirDepth=${#path[*]} |
| 1031 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1032 | local field=${path[${x}]} |
| 71353a5f |
1033 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break |
| 5dfac970 |
1034 | done |
| 71353a5f |
1035 | echo ${year} |
| 5dfac970 |
1036 | } |
| fc7e1e45 |
1037 | |
| 5dfac970 |
1038 | guessRunNumber() |
| 1039 | { |
| 1040 | #guess the run number from the path, pick the rightmost one |
| 71353a5f |
1041 | #works for /path/foo/000123456/bar/... |
| 1042 | #and /path/foo.run123456.bar |
| 1043 | local IFS="/." |
| 1044 | declare -a path=( ${1} ) |
| 5dfac970 |
1045 | local dirDepth=${#path[*]} |
| 1046 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1047 | local field=${path[${x}]} |
| 71353a5f |
1048 | field=${field/run/000} |
| 5dfac970 |
1049 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break |
| 1050 | done |
| 71353a5f |
1051 | echo ${runNumber} |
| 5dfac970 |
1052 | } |
| fc7e1e45 |
1053 | |
| cb473870 |
1054 | validateLog() |
| 1055 | { |
| 71353a5f |
1056 | log=${1} |
| cb473870 |
1057 | errorConditions=( |
| 71353a5f |
1058 | 'There was a crash' |
| 1059 | 'floating' |
| 1060 | 'error while loading shared libraries' |
| 1061 | 'std::bad_alloc' |
| 1062 | 's_err_syswatch_' |
| 1063 | 'Thread [0-9]* (Thread' |
| 1064 | 'AliFatal' |
| 1065 | 'core dumped' |
| 1066 | '\.C.*error:.*\.h: No such file' |
| 1067 | ) |
| 1068 | |
| 1069 | warningConditions=( |
| 1070 | 'This is serious !' |
| 1071 | 'rocVoltage out of range:' |
| cb473870 |
1072 | ) |
| 1073 | |
| 71353a5f |
1074 | local logstatus=0 |
| 1075 | local errorSummary="" |
| 1076 | local warningSummary="" |
| 1077 | |
| 1078 | for ((i=0; i<${#errorConditions[@]};i++)); do |
| 1079 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) |
| 1080 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1081 | errorSummary+=${tmp} |
| 1082 | done |
| 1083 | |
| 1084 | for ((i=0; i<${#warningConditions[@]};i++)); do |
| 1085 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) |
| 1086 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1087 | warningSummary+=${tmp} |
| 1088 | done |
| 1089 | |
| 1090 | if [[ -n ${errorSummary} ]]; then |
| 1091 | echo "${errorSummary}" |
| 1092 | return 1 |
| 1093 | fi |
| cb473870 |
1094 | |
| 71353a5f |
1095 | if [[ -n ${warningSummary} ]]; then |
| 1096 | echo "${warningSummary}" |
| 1097 | return 2 |
| 1098 | fi |
| 1099 | |
| 1100 | return 0 |
| cb473870 |
1101 | } |
| 1102 | |
| 5dfac970 |
1103 | summarizeLogs() |
| 1104 | { |
| 1105 | #print a summary of logs |
| 1106 | logFiles=( |
| 1107 | "*.log" |
| cb473870 |
1108 | "stdout" |
| 1109 | "stderr" |
| 5dfac970 |
1110 | ) |
| fc7e1e45 |
1111 | |
| cb473870 |
1112 | #check logs |
| 71353a5f |
1113 | local logstatus=0 |
| 5dfac970 |
1114 | for log in ${logFiles[*]}; do |
| 71353a5f |
1115 | finallog=${PWD%/}/${log} |
| 1116 | [[ ! -f ${log} ]] && continue |
| 1117 | errorSummary=$(validateLog ${log}) |
| 1118 | validationStatus=$? |
| 1119 | if [[ ${validationStatus} -eq 0 ]]; then |
| 5dfac970 |
1120 | #in pretend mode randomly report an error in rec.log some cases |
| 71353a5f |
1121 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then |
| 1122 | [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" |
| fc7e1e45 |
1123 | else |
| 71353a5f |
1124 | echo "${finallog} OK" |
| fc7e1e45 |
1125 | fi |
| 71353a5f |
1126 | elif [[ ${validationStatus} -eq 1 ]]; then |
| 1127 | echo "${finallog} BAD ${errorSummary}" |
| 1128 | elif [[ ${validationStatus} -eq 2 ]]; then |
| 1129 | echo "${finallog} OK MWAH ${errorSummary}" |
| 5dfac970 |
1130 | fi |
| 1131 | done |
| cb473870 |
1132 | |
| 1133 | #report core files |
| 71353a5f |
1134 | while read x; do |
| 1135 | echo ${x} |
| 1136 | chmod 644 ${x} |
| 1137 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log |
| 1138 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) |
| cb473870 |
1139 | |
| 71353a5f |
1140 | return ${logstatus} |
| 5dfac970 |
1141 | } |
| fc7e1e45 |
1142 | |
| 5dfac970 |
1143 | spitOutLocalOCDBaccessConfig() |
| 1144 | { |
| 1145 | umask 0002 |
| 71353a5f |
1146 | #find ${1} -name "*root" | \ |
| 1147 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ |
| 5dfac970 |
1148 | while read line |
| 1149 | do |
| 71353a5f |
1150 | local tmp=${line#${1}} |
| 5dfac970 |
1151 | echo ${tmp%/*} | \ |
| 71353a5f |
1152 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' |
| 5dfac970 |
1153 | done |
| 1154 | } |
| fc7e1e45 |
1155 | |
| 71353a5f |
1156 | goMakeLocalOCDBaccessConfig() |
| 5dfac970 |
1157 | { |
| 1158 | umask 0002 |
| 1159 | # make a script that sets the specific storages form all the root files produced by CPass0 |
| 71353a5f |
1160 | local localOCDBpathCPass0=${1} |
| 1161 | local OCDBpathPrefix=${2} |
| 1162 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." |
| 1163 | |
| 1164 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then |
| 1165 | tar xzf ${localOCDBpathCPass0} |
| 1166 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" |
| 5dfac970 |
1167 | fi |
| fc7e1e45 |
1168 | |
| 5dfac970 |
1169 | echo |
| 1170 | echo creating the specific storage script |
| 1171 | echo localOCDBaccessConfig.C |
| 71353a5f |
1172 | echo based on OCDB: ${localOCDBaccessConfig} |
| 5dfac970 |
1173 | echo |
| fc7e1e45 |
1174 | |
| 5dfac970 |
1175 | local tempLocalOCDB="" |
| 1176 | if [[ -f localOCDBaccessConfig.C ]]; then |
| 1177 | tempLocalOCDB=$(mktemp) |
| 71353a5f |
1178 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" |
| 1179 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} |
| 5dfac970 |
1180 | fi |
| fc7e1e45 |
1181 | |
| 5dfac970 |
1182 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
| 1183 | echo "{" >> localOCDBaccessConfig.C |
| 1184 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C |
| 71353a5f |
1185 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C |
| 1186 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C |
| 5dfac970 |
1187 | echo "}" >> localOCDBaccessConfig.C |
| fc7e1e45 |
1188 | |
| 71353a5f |
1189 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} |
| fc7e1e45 |
1190 | |
| 5dfac970 |
1191 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
| 1192 | echo |
| 1193 | echo "!!!!!!! CPass0 produced no OCDB entries" |
| 1194 | return 1 |
| 1195 | fi |
| 1196 | } |
| fc7e1e45 |
1197 | |
| cb473870 |
1198 | goMakeFilteredTrees() |
| 1199 | { |
| 71353a5f |
1200 | outputDir=${1} |
| 1201 | runNumber=${2} |
| cb473870 |
1202 | #get path to input list |
| 71353a5f |
1203 | inputListfiles=${3} |
| cb473870 |
1204 | #get scale number for tracks |
| 71353a5f |
1205 | filterT=${4} |
| cb473870 |
1206 | #get scale number for V0s |
| 71353a5f |
1207 | filterV=${5} |
| cb473870 |
1208 | #get OCDB path (optional) |
| 1209 | OCDBpath=${6} |
| 1210 | #get max number of files |
| 1211 | maxFiles=${7-"1000000"} |
| 1212 | #get offset of first file |
| 1213 | offsetFile=${8-"0"} |
| 1214 | #get max number of events |
| 1215 | maxEvents=${9-"30000000"} |
| 1216 | #get offset of first event |
| 1217 | offsetEvent=${10-"0"} |
| 1218 | configFile=${11-"benchmark.config"} |
| 1219 | esdFileName=${12-"AliESDs_Barrel.root"} |
| 71353a5f |
1220 | shift 12 |
| 1221 | parseConfig ${configFile} $@ |
| cb473870 |
1222 | |
| 1223 | commonOutputPath=${PWD} |
| 71353a5f |
1224 | doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done |
| 1225 | |
| cb473870 |
1226 | runpath=${commonOutputPath}/rundir_filtering_run${runNumber} |
| 1227 | |
| 71353a5f |
1228 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 1229 | |
| cb473870 |
1230 | mkdir -p ${outputDir} |
| 1231 | mkdir -p ${runpath} |
| 1232 | |
| 1233 | cd ${runpath} |
| 1234 | |
| 1235 | cat > filtering.log << EOF |
| 1236 | goMakeFilteredTrees config: |
| 71353a5f |
1237 | runpath=${runpath} |
| 1238 | outputDir=${outputDir} |
| 1239 | commonOutputPath=${commonOutputPath} |
| 1240 | ALICE_ROOT=${ALICE_ROOT} |
| 1241 | PATH=${PATH} |
| 1242 | offsetEvent=$offsetEvent |
| 1243 | configFile=$configFile |
| 1244 | esdFileName=$esdFileName |
| cb473870 |
1245 | EOF |
| 1246 | |
| 1247 | if [[ -z ${pretend} ]];then |
| 71353a5f |
1248 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log |
| cb473870 |
1249 | else |
| 71353a5f |
1250 | touch filtering.log FilterEvents_Trees.root |
| cb473870 |
1251 | fi |
| 1252 | pwd |
| 71353a5f |
1253 | /bin/ls |
| 1254 | touch ${doneFile} |
| 1255 | summarizeLogs >> ${doneFile} |
| cb473870 |
1256 | echo mv -f * ${outputDir} |
| 1257 | mv -f * ${outputDir} |
| 71353a5f |
1258 | [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} |
| cb473870 |
1259 | cd ${commonOutputPath} |
| 71353a5f |
1260 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| cb473870 |
1261 | } |
| 1262 | |
| 1263 | submit() |
| 1264 | { |
| 1265 | umask 0002 |
| 71353a5f |
1266 | [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1 |
| 1267 | JobID=${1} |
| 1268 | startID=${2} |
| 1269 | endID=${3} |
| 1270 | waitForJOBID=${4} |
| 1271 | command=${5} |
| 1272 | commandArgs=${6} |
| cb473870 |
1273 | |
| 71353a5f |
1274 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 |
| cb473870 |
1275 | |
| 1276 | newFarm=$(which qsub|grep "^/usr/bin/qsub") |
| 1277 | |
| 1278 | batchSystem="SGE" |
| 1279 | |
| 71353a5f |
1280 | if [[ -z "${newFarm}" ]] |
| cb473870 |
1281 | then |
| 1282 | #old LSF |
| 1283 | # submit it (as job array) |
| 71353a5f |
1284 | nFiles=$(( ${endID}-${startID}+1 )) |
| 1285 | while [ ${startID} -le ${nFiles} ] ; do |
| 1286 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then |
| 1287 | endID=$(expr ${startID} + 999) |
| cb473870 |
1288 | else |
| 71353a5f |
1289 | endID=${nFiles} |
| cb473870 |
1290 | fi |
| 1291 | if [[ ${waitForJOBID} -eq 0 ]]; then |
| 71353a5f |
1292 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1293 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1294 | else |
| 71353a5f |
1295 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1296 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1297 | fi |
| 71353a5f |
1298 | startID=$(expr ${endID} + 1) |
| cb473870 |
1299 | done |
| 1300 | else |
| 1301 | #new SGE farm |
| 1302 | if [[ ${waitForJOBID} =~ "000" ]]; then |
| 71353a5f |
1303 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| 1304 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| cb473870 |
1305 | else |
| 71353a5f |
1306 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| 1307 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| cb473870 |
1308 | fi |
| 1309 | fi |
| 1310 | } |
| 1311 | |
| 1312 | goSubmitBatch() |
| 5dfac970 |
1313 | { |
| 71353a5f |
1314 | if [[ $# -lt 3 ]]; then |
| 1315 | echo "minimal use:" |
| 1316 | echo " ${0} submit fileList productionID configFile" |
| 1317 | return 0 |
| 1318 | fi |
| 1319 | |
| 1320 | productionID=${1} |
| 1321 | inputList=${2} |
| 1322 | configFile=${3:-"benchmark.config"} |
| 1323 | configFile=$(readlink -f ${configFile}) |
| 1324 | shift 3 |
| 1325 | extraOpts="$@" |
| 1326 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
1327 | |
| 5dfac970 |
1328 | #redirect all output to submit.log |
| 1329 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" |
| 1330 | exec 7>&1 |
| 1331 | exec 1>submit_${productionID//"/"/_}.log 2>&1 |
| fc7e1e45 |
1332 | |
| 5dfac970 |
1333 | umask 0002 |
| 71353a5f |
1334 | echo ${0}" submit $*" |
| 1335 | if [[ -z "${inputList}" || -z "${productionID}" ]] |
| 5dfac970 |
1336 | then |
| fc7e1e45 |
1337 | echo |
| 71353a5f |
1338 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" |
| fc7e1e45 |
1339 | echo |
| 71353a5f |
1340 | return |
| fc7e1e45 |
1341 | fi |
| fc7e1e45 |
1342 | |
| 5dfac970 |
1343 | # check if config file is there |
| 71353a5f |
1344 | if [ ! -f ${configFile} ]; then |
| 1345 | echo "ERROR! Config File '${configFile}' not found" >&2 |
| 1346 | return |
| 5dfac970 |
1347 | else |
| 71353a5f |
1348 | echo "Using Config File: '${configFile}'" |
| fc7e1e45 |
1349 | fi |
| 1350 | |
| 5dfac970 |
1351 | # and print the configuration |
| 71353a5f |
1352 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
1353 | |
| 71353a5f |
1354 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there |
| 1355 | self=$(readlink -f "${0}") |
| 1356 | configPath=$(dirname ${configFile}) |
| 5dfac970 |
1357 | export commonOutputPath=${baseOutputDirectory}/${productionID} |
| 71353a5f |
1358 | |
| cb473870 |
1359 | mkdir -p ${commonOutputPath} |
| 71353a5f |
1360 | mkdir -p ${commonOutputPath}/logs |
| 1361 | mkdir -p ${commonOutputPath}/meta |
| 1362 | |
| 1363 | cp ${self} ${commonOutputPath} |
| 1364 | cp ${configFile} ${commonOutputPath} |
| 1365 | cp ${inputList} ${commonOutputPath} |
| cb473870 |
1366 | self=${commonOutputPath}/${self##*/} |
| 71353a5f |
1367 | chmod u+x ${self} |
| cb473870 |
1368 | configFile=${commonOutputPath}/${configFile##*/} |
| 1369 | inputList=${commonOutputPath}/${inputList##*/} |
| fc7e1e45 |
1370 | |
| 5dfac970 |
1371 | #convert to absolut pathnames |
| 71353a5f |
1372 | inputList=$(readlink -f "${inputList}") |
| 5dfac970 |
1373 | #make list of runs |
| 71353a5f |
1374 | if [[ -z ${runNumber} ]]; then |
| 1375 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) |
| 5dfac970 |
1376 | else |
| 71353a5f |
1377 | listOfRuns=${runNumber} |
| 5dfac970 |
1378 | fi |
| fc7e1e45 |
1379 | |
| 71353a5f |
1380 | alirootSource=$(readlink -f "${alirootSource}") |
| fc7e1e45 |
1381 | |
| 5dfac970 |
1382 | echo "" |
| 1383 | echo "### BEGIN CONFIGURATION ###" |
| 1384 | echo "" |
| 1385 | echo "GENERAL:" |
| 1386 | echo "" |
| 71353a5f |
1387 | echo " productionID: ${productionID}" |
| 1388 | echo " batchCommand: ${batchCommand}" |
| 1389 | echo " batchFlags: ${batchFlags}" |
| 1390 | echo " alirootSource: ${alirootSource}" |
| 1391 | echo " ALICE_ROOT: ${ALICE_ROOT}" |
| 1392 | echo " ALIROOT_RELEASE: ${ALICE_RELEASE}" |
| 1393 | echo " inputList: ${inputList}" |
| 1394 | echo " configPath: ${configPath}" |
| 1395 | echo " commonOutputPath: ${commonOutputPath}" |
| 1396 | echo " defaultOCDB: ${defaultOCDB}" |
| 1397 | echo " autoOCDB: ${autoOCDB}" |
| 1398 | echo " recoTriggerOptions: ${recoTriggerOptions}" |
| 5dfac970 |
1399 | echo " runs:" |
| 1400 | echo " ${listOfRuns[*]}" |
| 1401 | echo "" |
| 1402 | echo "THE TRAIN WILL RUN:" |
| fc7e1e45 |
1403 | |
| 71353a5f |
1404 | if [ ${runCPass0reco} -eq 1 ]; then |
| 5dfac970 |
1405 | echo " Pass0 - Recontruction" |
| 1406 | fi |
| fc7e1e45 |
1407 | |
| 71353a5f |
1408 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1409 | echo " Pass0 - merging and OCDB export" |
| 1410 | fi |
| fc7e1e45 |
1411 | |
| 71353a5f |
1412 | if [ ${runCPass1reco} -eq 1 ]; then |
| 5dfac970 |
1413 | echo " Pass1 - Recontruction" |
| 1414 | fi |
| 71353a5f |
1415 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1416 | echo " Pass1 - merging and OCDB export" |
| 1417 | fi |
| fc7e1e45 |
1418 | |
| 5dfac970 |
1419 | echo "" |
| 1420 | echo "LIMITS:" |
| 71353a5f |
1421 | echo " max. Events/Chunk: ${nEvents}" |
| 1422 | echo " max. Number of Chunks per Run: ${nMaxChunks}" |
| 5dfac970 |
1423 | echo "" |
| 1424 | echo "### END CONFIGURATION ###" |
| 1425 | echo "" |
| fc7e1e45 |
1426 | |
| 36b5de1a |
1427 | |
| 5dfac970 |
1428 | # check if input file is there |
| 71353a5f |
1429 | if [ ! -f ${inputList} ]; then |
| 1430 | echo "ERROR! Input List '${inputList}' not found" >&2 |
| 1431 | return |
| 5dfac970 |
1432 | fi |
| 1433 | |
| 1434 | # define jobid (for dependent jobs) |
| cb473870 |
1435 | date=$(date +%Y_%m_%d_%H%M%S) |
| 5dfac970 |
1436 | #for each run we submit one jobarray: |
| fc7e1e45 |
1437 | for runNumber in ${listOfRuns[*]}; do |
| cb473870 |
1438 | |
| 71353a5f |
1439 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
1440 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| cb473870 |
1441 | |
| 1442 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" |
| 1443 | JOBID1="p0_${JOBpostfix}" |
| 1444 | JOBID1wait="w0_${JOBpostfix}" |
| 1445 | JOBID2="m0_${JOBpostfix}" |
| 1446 | JOBID2wait="wm0_${JOBpostfix}" |
| 1447 | JOBID3="op0_${JOBpostfix}" |
| 1448 | JOBID3wait="wop0_${JOBpostfix}" |
| 1449 | JOBID4="p1_${JOBpostfix}" |
| 1450 | JOBID4wait="w1_${JOBpostfix}" |
| 1451 | JOBID5="m1_${JOBpostfix}" |
| 1452 | JOBID5wait="wm1_${JOBpostfix}" |
| 1453 | JOBID6="s1_${JOBpostfix}" |
| 1454 | JOBID6wait="ws1_${JOBpostfix}" |
| 1455 | JOBID7="QA_${JOBpostfix}" |
| 1456 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" |
| 1457 | JOBfilterESDcpass1="fp1_${JOBpostfix}" |
| 1458 | LASTJOB="000" |
| 1459 | |
| 71353a5f |
1460 | oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList}) |
| cb473870 |
1461 | |
| 1462 | currentDefaultOCDB=${defaultOCDB} |
| 1463 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 1464 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
1465 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) |
| cb473870 |
1466 | fi |
| 1467 | |
| 71353a5f |
1468 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" |
| 1469 | |
| 1470 | ############################################################################### |
| 1471 | #run one chunk with valgrind: |
| 1472 | if [[ -n ${runValgrind} ]]; then |
| 1473 | [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2 |
| 1474 | [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" |
| 1475 | submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}" |
| 1476 | fi |
| fc7e1e45 |
1477 | |
| 5dfac970 |
1478 | ################################################################################ |
| 1479 | ################################################################################ |
| 1480 | # run the CPass0 if requested |
| fc7e1e45 |
1481 | |
| 71353a5f |
1482 | if [ ${runCPass0reco} -eq 1 ]; then |
| fc7e1e45 |
1483 | |
| fc7e1e45 |
1484 | echo |
| 71353a5f |
1485 | echo "starting CPass0... for run ${runNumber}" |
| fc7e1e45 |
1486 | echo |
| fc7e1e45 |
1487 | |
| 5dfac970 |
1488 | # create directory and copy all files that are needed |
| cb473870 |
1489 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" |
| 71353a5f |
1490 | mkdir -p ${targetDirectory} |
| 1491 | |
| cb473870 |
1492 | filesCPass0=( |
| 71353a5f |
1493 | "${configPath}/runCPass0.sh" |
| 1494 | "${configPath}/recCPass0.C" |
| 1495 | "${configPath}/runCalibTrain.C" |
| 1496 | "${configPath}/localOCDBaccessConfig.C" |
| 1497 | "${configPath}/OCDB*.root" |
| 1498 | "${configPath}/sim.C" |
| 1499 | "${configPath}/rec.C" |
| 1500 | "${configPath}/Config.C" |
| cb473870 |
1501 | ) |
| 1502 | for file in ${filesCPass0[*]}; do |
| 1503 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1504 | done |
| fc7e1e45 |
1505 | |
| 71353a5f |
1506 | localInputList=${targetDirectory}/${inputList##*/} |
| 1507 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 5dfac970 |
1508 | # limit nFiles to nMaxChunks |
| 71353a5f |
1509 | nFiles=$(wc -l < ${localInputList}) |
| 1510 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 |
| 1511 | echo "raw files in list: ${nFiles}" |
| 1512 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1513 | nFiles=${nMaxChunks} |
| 5dfac970 |
1514 | fi |
| 71353a5f |
1515 | echo "raw files to process: ${nFiles}" |
| 1516 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1517 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1518 | nFilesToWaitFor=${nFiles} |
| 5dfac970 |
1519 | else |
| 71353a5f |
1520 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| 5dfac970 |
1521 | fi |
| 71353a5f |
1522 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1523 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
1524 | |
| 71353a5f |
1525 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}" |
| fc7e1e45 |
1526 | |
| 5dfac970 |
1527 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 71353a5f |
1528 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'" |
| 1529 | LASTJOB=${JOBID1wait} |
| fc7e1e45 |
1530 | |
| 5dfac970 |
1531 | fi #end running CPass0 |
| 1532 | ################################################################################ |
| fc7e1e45 |
1533 | |
| fc7e1e45 |
1534 | |
| 5dfac970 |
1535 | ################################################################################ |
| 1536 | # submit merging of CPass0, depends on the reconstruction |
| fc7e1e45 |
1537 | |
| 71353a5f |
1538 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
1539 | |
| 5dfac970 |
1540 | echo |
| 71353a5f |
1541 | echo "submit CPass0 merging for run ${runNumber}" |
| 5dfac970 |
1542 | echo |
| fc7e1e45 |
1543 | |
| cb473870 |
1544 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" |
| 71353a5f |
1545 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
1546 | |
| cb473870 |
1547 | #copy the scripts |
| 1548 | filesMergeCPass0=( |
| 71353a5f |
1549 | "${configPath}/OCDB.root" |
| 1550 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 1551 | "${configPath}/mergeMakeOCDB.sh" |
| 1552 | "${configPath}/localOCDBaccessConfig.C" |
| 1553 | "${configPath}/mergeByComponent.C" |
| 1554 | "${configPath}/makeOCDB.C" |
| 1555 | "${configPath}/merge.C" |
| cb473870 |
1556 | ) |
| 1557 | for file in ${filesMergeCPass0[*]}; do |
| 1558 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1559 | done |
| 1560 | |
| 71353a5f |
1561 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}" |
| 1562 | LASTJOB=${JOBID2} |
| fc7e1e45 |
1563 | |
| 71353a5f |
1564 | if [[ -n ${generateMC} ]]; then |
| 1565 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done" |
| 1566 | LASTJOB="mrl${JOBpostfix}" |
| 1567 | fi |
| fc7e1e45 |
1568 | |
| 5dfac970 |
1569 | echo |
| fc7e1e45 |
1570 | fi |
| 5dfac970 |
1571 | # end of merging CPass0 |
| 1572 | ################################################################################ |
| fc7e1e45 |
1573 | |
| 5dfac970 |
1574 | ################################################################################ |
| 1575 | ################################################################################ |
| 1576 | # run the CPass1 if requested |
| fc7e1e45 |
1577 | |
| 71353a5f |
1578 | if [ ${runCPass1reco} -eq 1 ]; then |
| fc7e1e45 |
1579 | |
| cb473870 |
1580 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" |
| 71353a5f |
1581 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done |
| fc7e1e45 |
1582 | |
| 5dfac970 |
1583 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
| 71353a5f |
1584 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" |
| fc7e1e45 |
1585 | |
| 5dfac970 |
1586 | echo |
| 71353a5f |
1587 | echo "starting CPass1... for run ${runNumber}" |
| 5dfac970 |
1588 | echo |
| 1589 | |
| 1590 | # create directory and copy all files that are needed |
| 71353a5f |
1591 | mkdir -p ${targetDirectory} |
| 5dfac970 |
1592 | |
| cb473870 |
1593 | filesCPass1=( |
| 71353a5f |
1594 | "${configPath}/runCPass1.sh" |
| 1595 | "${configPath}/recCPass1.C" |
| 1596 | "${configPath}/recCPass1_OuterDet.C" |
| 1597 | "${configPath}/runCalibTrain.C" |
| 1598 | "${configPath}/QAtrain_duo.C" |
| 1599 | "${configPath}/localOCDBaccessConfig.C" |
| 1600 | "${configPath}/OCDB.root" |
| cb473870 |
1601 | ) |
| 1602 | for file in ${filesCPass1[*]}; do |
| 1603 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1604 | done |
| 5dfac970 |
1605 | |
| 71353a5f |
1606 | if [[ -n ${generateMC} ]]; then |
| 1607 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list |
| 1608 | else |
| 1609 | localInputList=${targetDirectory}/${inputList##*/} |
| 1610 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 1611 | fi |
| 5dfac970 |
1612 | # limit nFiles to nMaxChunks |
| 71353a5f |
1613 | nFiles=$(wc -l < ${localInputList}) |
| 1614 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue |
| 1615 | echo "raw files in list: ${nFiles}" |
| 1616 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1617 | nFiles=${nMaxChunks} |
| 5dfac970 |
1618 | fi |
| 71353a5f |
1619 | echo "raw files to process: ${nFiles}" |
| 1620 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1621 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1622 | nFilesToWaitFor=${nFiles} |
| fc7e1e45 |
1623 | else |
| 71353a5f |
1624 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| fc7e1e45 |
1625 | fi |
| 71353a5f |
1626 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1627 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
1628 | |
| 71353a5f |
1629 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}" |
| fc7e1e45 |
1630 | |
| 5dfac970 |
1631 | ################################################################################ |
| 1632 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 71353a5f |
1633 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'" |
| 1634 | LASTJOB=${JOBID4wait} |
| 5dfac970 |
1635 | ################################################################################ |
| fc7e1e45 |
1636 | |
| 5dfac970 |
1637 | echo |
| 1638 | fi #end running CPass1 |
| fc7e1e45 |
1639 | |
| 5dfac970 |
1640 | ################################################################################ |
| 1641 | # submit merging of CPass1, depends on the reconstruction |
| 71353a5f |
1642 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
1643 | |
| 5dfac970 |
1644 | echo |
| 71353a5f |
1645 | echo "submit CPass1 merging for run ${runNumber}" |
| 5dfac970 |
1646 | echo |
| fc7e1e45 |
1647 | |
| cb473870 |
1648 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" |
| 71353a5f |
1649 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done |
| 1650 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
1651 | |
| cb473870 |
1652 | # copy files |
| 1653 | filesMergeCPass1=( |
| 71353a5f |
1654 | "${configPath}/OCDB.root" |
| 1655 | "${configPath}/localOCDBaccessConfig.C" |
| 1656 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 1657 | "${configPath}/mergeByComponent.C" |
| 1658 | "${configPath}/makeOCDB.C" |
| 1659 | "${configPath}/merge.C" |
| 1660 | "${configPath}/mergeMakeOCDB.sh" |
| 1661 | "${configPath}/QAtrain_duo.C" |
| cb473870 |
1662 | ) |
| 1663 | for file in ${filesMergeCPass1[*]}; do |
| 1664 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1665 | done |
| 1666 | |
| 71353a5f |
1667 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}" |
| 1668 | LASTJOB=${JOBID5} |
| 5dfac970 |
1669 | echo |
| 1670 | fi |
| 1671 | |
| cb473870 |
1672 | ############################## |
| 71353a5f |
1673 | if [ ${runESDfiltering} -eq 1 ]; then |
| 1674 | rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list |
| 1675 | rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done |
| 1676 | echo |
| 1677 | echo submitting filtering for run ${runNumber} |
| 1678 | echo |
| 1679 | submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done " |
| 1680 | submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}" |
| 1681 | LASTJOB=${JOBfilterESDcpass1} |
| cb473870 |
1682 | fi |
| 1683 | |
| 5dfac970 |
1684 | done |
| 1685 | |
| 71353a5f |
1686 | ################################################################################# |
| 1687 | ################################################################################# |
| 1688 | if [ ${runESDfiltering} -eq 1 ]; then |
| 1689 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}" |
| 1690 | else |
| 1691 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}" |
| cb473870 |
1692 | fi |
| 71353a5f |
1693 | LASTJOB=${JOBID5wait} |
| cb473870 |
1694 | |
| 71353a5f |
1695 | [[ -z ${runMakeSummary} ]] && runMakeSummary=0 |
| 1696 | if [ ${runMakeSummary} -eq 1 ]; then |
| fc7e1e45 |
1697 | echo |
| 5dfac970 |
1698 | echo "submit make a summary" |
| 1699 | echo |
| 1700 | |
| 71353a5f |
1701 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}" |
| 1702 | LASTJOB=${JOBID6} |
| 5dfac970 |
1703 | fi |
| 71353a5f |
1704 | ################################################################################# |
| 5dfac970 |
1705 | |
| 5dfac970 |
1706 | #restore stdout |
| 1707 | exec 1>&7 7>&- |
| 1708 | echo "jobs submitted." |
| fc7e1e45 |
1709 | } |
| 1710 | |
| cb473870 |
1711 | goWaitForOutput() |
| 1712 | { |
| 1713 | umask 0002 |
| 71353a5f |
1714 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 |
| 1715 | echo searchPath=${1} |
| 1716 | echo fileName=${2} |
| 1717 | echo numberOfFiles=${3} |
| 1718 | echo maxSecondsToWait=${4} |
| 1719 | searchPath=${1} |
| 1720 | fileName=${2} |
| 1721 | numberOfFiles=${3} |
| 1722 | maxSecondsToWait=${4} |
| 1723 | extraFindOptions=${5} |
| 1724 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}" |
| 1725 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) |
| cb473870 |
1726 | while true; do |
| 71353a5f |
1727 | #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l) |
| cb473870 |
1728 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) |
| 71353a5f |
1729 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" |
| 1730 | [[ ${n} -ge ${numberOfFiles} ]] && break |
| 1731 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break |
| cb473870 |
1732 | sleep 60 |
| 1733 | done |
| 1734 | echo "DONE! exiting..." |
| 1735 | } |
| 1736 | |
| 71353a5f |
1737 | mergeSysLogs() |
| 1738 | { |
| 1739 | outputFile=${1} |
| 1740 | shift |
| 1741 | inputFiles="$@" |
| 1742 | i=0 |
| 1743 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi |
| 1744 | while read x; do |
| 1745 | runNumber=$(guessRunNumber ${x}) |
| 1746 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue |
| 1747 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} |
| 1748 | (( i++ )) |
| 1749 | done < <(ls -1 ${inputFiles}) > ${outputFile} |
| 1750 | } |
| 1751 | |
| 1752 | goMakeMergedSummaryTree() |
| 1753 | { |
| 1754 | # create list of calibration entries |
| 1755 | # takes no arguments, just run it in the base output |
| 1756 | # directory with the following files in the working directory |
| 1757 | # |
| 1758 | # Calibration file lists: |
| 1759 | # cpass0.dcsTree.list, cpass1.dcsTree.list |
| 1760 | # QA trending root file: |
| 1761 | # trending.root |
| 1762 | # |
| 1763 | # Production infoo ascii files: |
| 1764 | # summary_pass0.tree |
| 1765 | # summary_pass1.tree |
| 1766 | # |
| 1767 | |
| 1768 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 |
| 1769 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 |
| 1770 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 |
| 1771 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 |
| 1772 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 |
| 1773 | |
| 1774 | #first, dump the C macro to file |
| 1775 | cat << EOF > mergeTree.C |
| 1776 | // |
| 1777 | // Merge summary information |
| 1778 | // Following files are expected to be in the working directory |
| 1779 | // |
| 1780 | // Calibration file lists: |
| 1781 | // cpass0.dcsTree.list, cpass1.dcsTree.list |
| 1782 | // QA trending root file: |
| 1783 | // trending.root |
| 1784 | // |
| 1785 | // Production infoo ascii files: |
| 1786 | // summary_pass0.tree |
| 1787 | // summary_pass1.tree |
| 1788 | // |
| 1789 | void mergeTree(){ |
| 1790 | // |
| 1791 | // |
| 1792 | // |
| 1793 | // Calibration values dump |
| 1794 | // |
| 1795 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); |
| 1796 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); |
| 1797 | // |
| 1798 | // Calibration status dump |
| 1799 | // |
| 1800 | TFile *fprod = TFile::Open("fproduction.root","recreate"); |
| 1801 | TTree tree0, tree1; |
| 1802 | tree0.ReadFile("summary_pass0.tree"); |
| 1803 | tree1.ReadFile("summary_pass1.tree"); |
| 1804 | tree0.Write("CPass0"); |
| 1805 | tree1.Write("CPass1"); |
| 1806 | fprod->Close(); |
| 1807 | // |
| 1808 | // |
| 1809 | // |
| 1810 | TString stringSetup=""; |
| 1811 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // |
| 1812 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // |
| 1813 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // |
| 1814 | // |
| 1815 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // |
| 1816 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // |
| 1817 | // |
| 1818 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); |
| 1819 | } |
| 1820 | EOF |
| 1821 | |
| 1822 | aliroot -b -q "mergeTree.C" > mergeTrees.log |
| 1823 | return $? |
| 1824 | } |
| 1825 | |
| 1826 | goMakeSummary() |
| 1827 | { |
| 1828 | #all the final stuff goes in here for ease of use: |
| 1829 | # summary logs |
| 1830 | # qa plot making |
| 1831 | # final file lists |
| 1832 | |
| 1833 | configFile=${1} |
| 1834 | shift 1 |
| 1835 | extraOpts="$@" |
| 1836 | log="summary.log" |
| 1837 | productionID="qa" |
| 1838 | parseConfig ${configFile} ${extraOpts} |
| 1839 | |
| 1840 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 1841 | |
| 1842 | exec &> >(tee ${log}) |
| 1843 | |
| 1844 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return |
| 1845 | |
| 1846 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 1847 | |
| 1848 | #summarize the global stuff |
| 1849 | echo "env script: ${alirootSource} ${alirootEnv}" |
| 1850 | echo "\$ALICE_ROOT=${ALICE_ROOT}" |
| 1851 | echo "commonOutputPath=${commonOutputPath}" |
| 1852 | |
| 1853 | #summarize the stacktraces |
| 1854 | awk ' |
| 1855 | BEGIN { |
| 1856 | print "frame/I:method/C:line/C:cpass/I:aliroot/I"; |
| 1857 | RS="#[0-9]*"; |
| 1858 | aliroot=0; |
| 1859 | } |
| 1860 | { |
| 1861 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
| 1862 | gsub("#","",RT); |
| 1863 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot |
| 1864 | } |
| 1865 | ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree |
| 1866 | awk ' |
| 1867 | BEGIN { |
| 1868 | RS="#[0-9]*"; |
| 1869 | aliroot=0; |
| 1870 | } |
| 1871 | { |
| 1872 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
| 1873 | gsub("#","",RT); |
| 1874 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot |
| 1875 | } |
| 1876 | ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree |
| 1877 | |
| 1878 | echo total numbers for the production: |
| 1879 | echo |
| 1880 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 1881 | /^calibfile/ {nFiles++;} |
| 1882 | /core dumped/ {nCore++i;} |
| 1883 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null |
| 1884 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1885 | /\/rec.log OK/ {nOK++;} |
| 1886 | /\/rec.log BAD/ {nBAD++;} |
| 1887 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| 1888 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null |
| 1889 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1890 | /\/calib.log OK/ {nOK++;} |
| 1891 | /\/calib.log BAD/ {nBAD++;} |
| 1892 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null |
| 1893 | |
| 1894 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1895 | /merge.log OK/ {nOK++;} |
| 1896 | /merge.log BAD/ {nBAD++;} |
| 1897 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null |
| 1898 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1899 | /ocdb.log OK/ {nOK++;} |
| 1900 | /ocdb.log BAD/ {nBAD++;} |
| 1901 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null |
| 1902 | |
| 1903 | echo |
| 1904 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 1905 | /^calibfile/ {nFiles++;} |
| 1906 | /core dumped/ {nCore++;} |
| 1907 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null |
| 1908 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1909 | /\/rec.log OK/ {nOK++;} |
| 1910 | /\/rec.log BAD/ {nBAD++;} |
| 1911 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| 1912 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null |
| 1913 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1914 | /\/calib.log OK/ {nOK++;} |
| 1915 | /\/calib.log BAD/ {nBAD++;} |
| 1916 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null |
| 1917 | |
| 1918 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1919 | /merge.log OK/ {nOK++;} |
| 1920 | /merge.log BAD/ {nBAD++;} |
| 1921 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null |
| 1922 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1923 | /ocdb.log OK/ {nOK++;} |
| 1924 | /ocdb.log BAD/ {nBAD++;} |
| 1925 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null |
| 1926 | |
| 1927 | echo |
| 1928 | echo per run stats: |
| 1929 | /bin/ls -1 meta/merge.cpass0.run*.done | while read x |
| 1930 | do |
| 1931 | dir=$(goPrintValues calibfile - ${x}) |
| 1932 | runNumber=$(guessRunNumber ${dir}) |
| 1933 | [[ -z ${runNumber} ]] && continue |
| 1934 | |
| 1935 | if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then |
| 1936 | statusCPass0=( $( |
| 1937 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} |
| 1938 | /\/rec.log OK/ {nOKrec++;} |
| 1939 | /\/rec.log BAD/ {nBADrec++;} |
| 1940 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} |
| 1941 | /stderr OK/ {nOKstderr++;} |
| 1942 | /\/calib.log OK/ {nOKcalib++;} |
| 1943 | /\/calib.log BAD/ {nBADcalib++} |
| 1944 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null |
| 1945 | ) ) |
| 1946 | fi |
| 1947 | |
| 1948 | if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then |
| 1949 | statusCPass1=( $( |
| 1950 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} |
| 1951 | /\/rec.log OK/ {nOKrec++;} |
| 1952 | /\/rec.log BAD/ {nBADrec++;} |
| 1953 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} |
| 1954 | /stderr OK/ {nOKstderr++;} |
| 1955 | /\/calib.log OK/ {nOKcalib++;} |
| 1956 | /\/calib.log BAD/ {nBADcalib++} |
| 1957 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} |
| 1958 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} |
| 1959 | /\/qa_outer.log OK/ {nQAouterOK++;} |
| 1960 | /\/qa_outer.log BAD/ {nQAouterBAD++;} |
| 1961 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null |
| 1962 | ) ) |
| 1963 | fi |
| 1964 | |
| 1965 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) |
| 1966 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 1967 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 1968 | |
| 1969 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} |
| 1970 | done |
| 1971 | |
| 1972 | #make lists with output files - QA, trending, filtering and calibration |
| 1973 | ### wait for the merging of all runs to be over ### |
| 1974 | rm -f qa.list |
| 1975 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1976 | rm -f calib.list |
| 1977 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1978 | rm -f trending.list |
| 1979 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1980 | rm -f filtering.list |
| 1981 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null |
| 1982 | #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list |
| 1983 | rm -f cpass0.dcsTree.list |
| 1984 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null |
| 1985 | #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list |
| 1986 | rm -f cpass1.dcsTree.list |
| 1987 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1988 | |
| 1989 | #merge trending |
| 1990 | rm -f ${commonOutputPath}/trending_merged.root |
| 1991 | goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log |
| 1992 | |
| 1993 | goMakeSummaryTree ${commonOutputPath} 0 |
| 1994 | goMakeSummaryTree ${commonOutputPath} 1 |
| 1995 | |
| 1996 | goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log |
| 1997 | |
| 1998 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees |
| 1999 | goMakeMergedSummaryTree |
| 2000 | |
| 2001 | #if set, email the summary |
| 2002 | [[ -n ${mailSummaryTo} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${mailSummaryTo} |
| 2003 | |
| 2004 | return 0 |
| 2005 | } |
| 2006 | |
| 2007 | goMakeSummaryTree() |
| 2008 | { |
| 2009 | if [[ $# -lt 1 ]] ; then |
| 2010 | return |
| 2011 | fi |
| 2012 | #1. define vars/arrays |
| 2013 | DIR=${1} #use input or exec in current dir |
| 2014 | pass=${2-"0"} #pass from input |
| 2015 | outfile="summary_pass${pass}.tree" |
| 2016 | Ncolumns=0 |
| 2017 | test -f ${outfile} && : >${outfile} |
| 2018 | errfile=${outfile/tree/err} |
| 2019 | test -f ${errfile} && : >${errfile} |
| 2020 | |
| 2021 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) |
| 2022 | Ncounter=${#counterString[@]} |
| 2023 | |
| 2024 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) |
| 2025 | Nstatus=${#statusString[@]} |
| 2026 | |
| 2027 | |
| 2028 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) |
| 2029 | Nrates=${#ratesString[@]} |
| 2030 | |
| 2031 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) |
| 2032 | Nruns=${#runs[@]} |
| 2033 | |
| 2034 | echo -n runnumber/I >>${outfile} |
| 2035 | echo -n :cpass${pass}status/I >>${outfile} |
| 2036 | echo -n :cpass${pass}QAstatus/I >>${outfile} |
| 2037 | for i in ${ratesString[@]}; do |
| 2038 | echo -n :${i}OK/I >>${outfile} |
| 2039 | echo -n :${i}BAD/I >>${outfile} |
| 2040 | |
| 2041 | done |
| 2042 | for i in ${counterString[@]} ${statusString[@]} ; do |
| 2043 | echo -n :${i}/I >>${outfile} |
| 2044 | done |
| 2045 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) |
| 2046 | echo >> ${outfile} |
| 2047 | |
| 2048 | #2. loop runs |
| 2049 | |
| 2050 | for runnumber in ${runs[@]} ; do |
| 2051 | |
| 2052 | |
| 2053 | |
| 2054 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" |
| 2055 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" |
| 2056 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') |
| 2057 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then |
| 2058 | echo "${filemerge} does not exist!" >>${errfile} |
| 2059 | continue |
| 2060 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then |
| 2061 | echo "${filejob} does not exist!" >>${errfile} |
| 2062 | echo -n ${runnumber} >> ${outfile} |
| 2063 | for i in $(seq ${Ncolumns}) ; do |
| 2064 | echo -n "-1" >> ${outfile} |
| 2065 | done |
| 2066 | echo >> ${outfile} |
| 2067 | continue |
| 2068 | fi |
| 2069 | echo -n ${runnumber} >> ${outfile} |
| 2070 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' |
| 2071 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) |
| 2072 | echo -n " ${passStatus}" >> ${outfile} |
| 2073 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) |
| 2074 | echo -n " ${qaStatus}" >> ${outfile} |
| 2075 | |
| 2076 | |
| 2077 | #fill OK/BAD rates |
| 2078 | for i in $(seq 0 $((${Nrates}-1))) ; do |
| 2079 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) |
| 2080 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) |
| 2081 | |
| 2082 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then |
| 2083 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) |
| 2084 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) |
| 2085 | fi |
| 2086 | echo -n " ${var1}" >> ${outfile} |
| 2087 | echo -n " ${var2}" >> ${outfile} |
| 2088 | done |
| 2089 | |
| 2090 | if [[ -f ${fileOCDB} ]] ; then |
| 2091 | #fill counter |
| 2092 | for i in $(seq 0 $((${Ncounter}-1))) ; do |
| 2093 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) |
| 2094 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2095 | done |
| 2096 | |
| 2097 | #fill status |
| 2098 | for i in $(seq 0 $((${Nstatus}-1))) ; do |
| 2099 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) |
| 2100 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2101 | done |
| 2102 | fi |
| 2103 | echo >> ${outfile} |
| 2104 | done |
| 2105 | |
| 2106 | } |
| 2107 | |
| 2108 | parseConfig() |
| 2109 | { |
| 2110 | configFile=${1} |
| 2111 | shift |
| 2112 | |
| 2113 | #first, source the config file |
| 2114 | if [ -f ${configFile} ]; then |
| 2115 | source ${configFile} |
| 2116 | else |
| 2117 | echo "config file ${configFile} not found!, skipping..." |
| 2118 | fi |
| 2119 | |
| 2120 | #then, parse the options as theya override the optionf from file |
| 2121 | while [[ -n ${1} ]]; do |
| 2122 | var="${1#--}" |
| 2123 | echo "extraOpts: exporting ${var}" |
| 2124 | eval "export ${var}" |
| 2125 | shift |
| 2126 | done |
| 2127 | |
| 2128 | #export the aliroot function if defined to override normal behaviour |
| 2129 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot |
| 2130 | } |
| 2131 | |
| 2132 | aliroot() |
| 2133 | { |
| 2134 | args="$@" |
| 2135 | if [[ -n ${valgrindCommand} ]]; then |
| 2136 | valgrindLogFile="cpu.txt" |
| 2137 | [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt" |
| 2138 | [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt" |
| 2139 | [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}" |
| 2140 | echo running ${valgrindCommand} aliroot ${args} |
| 2141 | ${valgrindCommand} aliroot ${args} |
| 2142 | else |
| 2143 | #to prevent an infinite recursion use "command aliroot" to disable |
| 2144 | #aliases and functions |
| 2145 | echo running command aliroot ${args} |
| 2146 | command aliroot ${args} |
| 2147 | fi |
| 2148 | } |
| 2149 | |
| 2150 | guessRunData() |
| 2151 | { |
| 2152 | #guess the period from the path, pick the rightmost one |
| 2153 | period="" |
| 2154 | runNumber="" |
| 2155 | year="" |
| 2156 | pass="" |
| 2157 | legoTrainRunNumber="" |
| 2158 | dataType="" |
| 2159 | |
| 2160 | local shortRunNumber="" |
| 2161 | local IFS="/" |
| 2162 | declare -a path=( $1 ) |
| 2163 | local dirDepth=$(( ${#path[*]}-1 )) |
| 2164 | i=0 |
| 2165 | #for ((x=${dirDepth};x>=0;x--)); do |
| 2166 | for ((x=0;x<=${dirDepth};x++)); do |
| 2167 | |
| 2168 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} |
| 2169 | local field=${path[${x}]} |
| 2170 | local fieldNext=${path[$((x+1))]} |
| 2171 | |
| 2172 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} |
| 2173 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} |
| 2174 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} |
| 2175 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} |
| 2176 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} |
| 2177 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} |
| 2178 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} |
| 2179 | (( i++ )) |
| 2180 | done |
| 2181 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} |
| 2182 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} |
| 2183 | |
| 2184 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; |
| 2185 | if [[ -z ${runNumber}} ]]; |
| 2186 | then |
| 2187 | #error condition |
| 2188 | return 1 |
| 2189 | else |
| 2190 | #ALL OK |
| 2191 | return 0 |
| 2192 | fi |
| 2193 | } |
| 2194 | |
| fc7e1e45 |
2195 | main "$@" |
git://git.uio.no / u / mrichter / AliRoot.git / blame