| fc7e1e45 |
1 | #!/bin/bash |
| 71353a5f |
2 | #include benchmark.config |
| 3 | |
| fc7e1e45 |
4 | # this script runs the CPass0/CPass1 train |
| 5 | # produced OCDB updates are local |
| 6 | |
| 7 | main() |
| 8 | { |
| 9 | #run in proper mode depending on the selection |
| 71353a5f |
10 | if [[ $# -lt 1 ]]; then |
| 11 | if [[ ! "${0}" =~ "bash" ]]; then |
| 12 | echo "uses makeflow:" |
| 13 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" |
| 14 | echo "uses a batch system (SGE):" |
| 15 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" |
| 16 | echo "extraOpts if non-empty override the config file, e.g.:" |
| 17 | echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10" |
| 18 | fi |
| 19 | return |
| 20 | fi |
| 21 | |
| 22 | #define some aliases - default is to call one of the functions directly |
| 23 | runMode=${1} |
| fc7e1e45 |
24 | umask 0002 |
| 25 | shift |
| 71353a5f |
26 | case ${runMode} in |
| fc7e1e45 |
27 | "CPass0") goCPass0 "$@";; |
| 28 | "CPass1") goCPass1 "$@";; |
| 71353a5f |
29 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; |
| fc7e1e45 |
30 | "MergeCPass0") goMergeCPass0 "$@";; |
| 31 | "MergeCPass1") goMergeCPass1 "$@";; |
| cb473870 |
32 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; |
| 71353a5f |
33 | "MakeSummary") goMakeSummary "$@";; |
| cb473870 |
34 | "run") goSubmitMakeflow "$@";; |
| 35 | "submit") goSubmitBatch "$@";; |
| fc7e1e45 |
36 | "test") goTest "$@";; |
| 71353a5f |
37 | "GenerateMakeflow") goGenerateMakeflow "$@";; |
| 38 | "PrintValues") goPrintValues "$@";; |
| cb473870 |
39 | "CreateQAplots") goCreateQAplots "$@";; |
| 40 | "WaitForOutput") goWaitForOutput "$@";; |
| 71353a5f |
41 | "Merge") goMerge "$@";; |
| 42 | *) |
| 43 | eval ${runMode} $@ |
| fc7e1e45 |
44 | ;; |
| 45 | esac |
| 46 | } |
| 47 | |
| 71353a5f |
48 | generateMC() |
| 49 | { |
| 50 | #generate one raw chunk in current directory |
| 51 | SEED=${JOB_ID}${SGE_TASK_ID} |
| 52 | export CONFIG_SEED=${SEED} |
| 53 | runNumber=${1} |
| 54 | OCDBpath=${2} |
| 55 | nEventsim=${3} |
| 56 | if [[ -n ${pretend} ]]; then |
| 57 | touch galice.root |
| 58 | else |
| 59 | if [[ -f sim.C && -f Config.C ]] ; then |
| 60 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 |
| 61 | mv syswatch.log simwatch.log |
| 62 | fi |
| 63 | fi |
| 64 | } |
| 65 | |
| fc7e1e45 |
66 | goCPass0() |
| 67 | { |
| 68 | umask 0002 |
| 69 | |
| 71353a5f |
70 | targetDirectory=${1} |
| 71 | inputList=${2} |
| 72 | nEvents=${3} |
| 73 | ocdbPath=${4} |
| 74 | configFile=${5} |
| 75 | runNumber=${6} |
| 76 | jobindex=${7} |
| 77 | shift 7 |
| 78 | parseConfig ${configFile} $@ |
| 79 | |
| 80 | #use the jobindex only if set and non-negative |
| 81 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 82 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 83 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 84 | fi |
| 85 | |
| 86 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 87 | doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" |
| 88 | |
| 89 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 90 | |
| 91 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 92 | ulimit -c unlimited |
| 93 | export ALIROOT_FORCE_COREDUMP |
| 94 | fi |
| fc7e1e45 |
95 | |
| cb473870 |
96 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 97 | #at the QA stage the pass number is guessed from the path stored here. |
| 98 | #The Format is: |
| 99 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 100 | export PRODUCTION_METADATA="OutputDir=cpass0" |
| 101 | |
| 5de3064b |
102 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 103 | infile=${inputList} |
| 104 | else |
| 105 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| 106 | fi |
| 71353a5f |
107 | chunkName=${infile##*/} |
| 71353a5f |
108 | |
| 109 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 110 | mkdir -p ${outputDir} |
| 111 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 |
| 112 | |
| 113 | #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} |
| 114 | runpath=${outputDir} |
| 115 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 116 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 71353a5f |
117 | mkdir -p ${runpath} |
| 118 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 |
| 119 | cd ${runpath} |
| fc7e1e45 |
120 | |
| 5de3064b |
121 | #runCPassX/C expects the raw chunk to be linked in the run dir |
| 122 | #despite it being accessed by the full path |
| 123 | ln -s ${infile} ${runpath}/${chunkName} |
| fc7e1e45 |
124 | |
| 71353a5f |
125 | #####MC |
| 126 | if [[ -n ${generateMC} ]]; then |
| 127 | olddir=${PWD} |
| 128 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} |
| 129 | simrunpath=${outputDirMC} |
| 130 | [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} |
| 131 | [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d) |
| 132 | mkdir -p ${outputDirMC} |
| 133 | mkdir -p ${simrunpath} |
| 134 | cd ${simrunpath} |
| 135 | |
| 136 | filesMC=( |
| 137 | "${commonOutputPath}/sim.C" |
| 138 | "${commonOutputPath}/rec.C" |
| 139 | "${commonOutputPath}/Config.C" |
| 140 | "${commonOutputPath}/OCDB_*.root" |
| 141 | ) |
| 142 | for file in ${filesMC[*]}; do |
| 143 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 144 | done |
| fc7e1e45 |
145 | |
| 71353a5f |
146 | generateMC ${runNumber} ${ocdbPath} ${nEvents} |
| fc7e1e45 |
147 | |
| 71353a5f |
148 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works |
| 149 | cd ${olddir} |
| 150 | |
| 151 | ln -s ${outputDirMC}/* ${runpath}/ |
| 152 | |
| 153 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! |
| 154 | infile="" |
| 155 | fi |
| 156 | ###### |
| 157 | |
| 158 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 |
| 159 | |
| 160 | logOutputDir=${runpath} |
| 161 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 162 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout |
| 163 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 164 | echo "${0} $*" |
| fc7e1e45 |
165 | |
| 166 | echo "#####################" |
| 167 | echo CPass0: |
| 168 | echo JOB setup |
| 71353a5f |
169 | echo nEvents ${nEvents} |
| 170 | echo runNumber ${runNumber} |
| 171 | echo ocdbPath ${ocdbPath} |
| 172 | echo infile ${infile} |
| 173 | echo chunkName ${chunkName} |
| 174 | echo jobindex ${jobindex} |
| 175 | echo recoTriggerOptions ${recoTriggerOptions} |
| 176 | echo targetDirectory ${targetDirectory} |
| 177 | echo commonOutputPath ${commonOutputPath} |
| 178 | echo doneFile ${doneFile} |
| 179 | echo runpath ${runpath} |
| 180 | echo outputDir ${outputDir} |
| 181 | echo ALICE_ROOT ${ALICE_ROOT} |
| 182 | echo PWD ${PWD} |
| fc7e1e45 |
183 | echo "########## ###########" |
| 184 | |
| 71353a5f |
185 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
186 | |
| 187 | filesCPass0=( |
| 71353a5f |
188 | "${commonOutputPath}/runCPass0.sh" |
| 189 | "${commonOutputPath}/recCPass0.C" |
| 190 | "${commonOutputPath}/runCalibTrain.C" |
| 191 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 192 | "${commonOutputPath}/OCDB.root" |
| 193 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
| 194 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" |
| 195 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" |
| fc7e1e45 |
196 | ) |
| 197 | |
| 198 | for file in ${filesCPass0[*]}; do |
| 199 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 200 | done |
| 201 | |
| 71353a5f |
202 | echo "this directory (${PWD}) contents:" |
| 203 | /bin/ls |
| fc7e1e45 |
204 | echo |
| 205 | chmod u+x runCPass0.sh |
| 206 | |
| 71353a5f |
207 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 208 | |
| 209 | if [[ -n ${postSetUpActionCPass0} ]]; then |
| 210 | echo "running ${postSetUpActionCPass0}" |
| 211 | eval ${postSetUpActionCPass0} |
| fc7e1e45 |
212 | fi |
| 213 | |
| 214 | #run CPass0 |
| 71353a5f |
215 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 216 | if [[ -n ${pretend} ]]; then |
| cb473870 |
217 | touch AliESDs.root |
| 218 | touch AliESDfriends.root |
| fc7e1e45 |
219 | touch AliESDfriends_v1.root |
| 220 | touch rec.log |
| 221 | touch calib.log |
| 222 | else |
| 71353a5f |
223 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| 224 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| fc7e1e45 |
225 | fi |
| 226 | |
| fc7e1e45 |
227 | #move stuff to final destination |
| 71353a5f |
228 | echo "this directory (${PWD}) contents:" |
| 229 | /bin/ls |
| fc7e1e45 |
230 | echo |
| 231 | |
| 71353a5f |
232 | echo rm -f ./${chunkName} |
| 233 | rm -f ./${chunkName} |
| 234 | echo "cp --recursive ${runpath}/* ${outputDir}" |
| 235 | cp -p --recursive ${runpath}/* ${outputDir} |
| fc7e1e45 |
236 | echo |
| 5ed7c82c |
237 | |
| 238 | #validate CPass0 |
| 239 | cd ${outputDir} |
| 71353a5f |
240 | touch ${doneFile} |
| 241 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile} |
| 242 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} |
| 243 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile} |
| 244 | echo "dir ${outputDir}" >> ${doneFile} |
| 245 | summarizeLogs >> ${doneFile} |
| fc7e1e45 |
246 | |
| 71353a5f |
247 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| 5de3064b |
248 | return 0 |
| fc7e1e45 |
249 | } |
| 250 | |
| 251 | goCPass1() |
| 252 | { |
| 253 | umask 0002 |
| 254 | |
| 71353a5f |
255 | targetDirectory=${1} |
| 256 | inputList=${2} |
| 257 | nEvents=${3} |
| 258 | ocdbPath=${4} |
| 259 | configFile=${5} |
| 260 | runNumber=${6} |
| 261 | jobindex=${7} |
| 262 | shift 7 |
| 263 | parseConfig ${configFile} $@ |
| 264 | |
| 265 | #use the jobindex only if set and non-negative |
| 266 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 267 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 268 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 269 | fi |
| 270 | |
| 271 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 272 | doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" |
| 273 | |
| 274 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 275 | |
| 276 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 277 | ulimit -c unlimited |
| 278 | export ALIROOT_FORCE_COREDUMP |
| 279 | fi |
| fc7e1e45 |
280 | |
| cb473870 |
281 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 282 | #at the QA stage the pass number is guessed from the path stored here. |
| 283 | #The Format is: |
| 284 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 285 | export PRODUCTION_METADATA="OutputDir=cpass1" |
| 286 | |
| 71353a5f |
287 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 |
| fc7e1e45 |
288 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 71353a5f |
289 | infile=${inputList} |
| fc7e1e45 |
290 | else |
| 71353a5f |
291 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| fc7e1e45 |
292 | fi |
| fc7e1e45 |
293 | chunkName=${infile##*/} |
| 5de3064b |
294 | |
| 71353a5f |
295 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 296 | mkdir -p ${outputDir} |
| 297 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 |
| fc7e1e45 |
298 | |
| 71353a5f |
299 | #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} |
| 300 | runpath=${outputDir} |
| 301 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 302 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 303 | |
| 304 | #MC |
| 305 | if [[ "${infile}" =~ galice\.root ]]; then |
| 306 | ln -s ${inputList%/*}/* ${runpath} |
| 307 | infile="" |
| 308 | fi |
| fc7e1e45 |
309 | |
| 310 | #init the running path |
| 71353a5f |
311 | mkdir -p ${runpath} |
| 312 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 |
| 313 | cd ${runpath} |
| fc7e1e45 |
314 | |
| 71353a5f |
315 | #this is needed for runCPass1.sh |
| 316 | ln -s ${infile} ${runpath}/${chunkName} |
| fc7e1e45 |
317 | |
| 71353a5f |
318 | logOutputDir=${runpath} |
| 319 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 320 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout |
| 321 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 322 | echo "${0} $*" |
| fc7e1e45 |
323 | |
| 324 | echo "#####################" |
| 325 | echo CPass1: |
| 326 | echo JOB setup |
| 71353a5f |
327 | echo nEvents ${nEvents} |
| 328 | echo runNumber ${runNumber} |
| 329 | echo ocdbPath ${ocdbPath} |
| 330 | echo infile ${infile} |
| 331 | echo chunkName ${chunkName} |
| 332 | echo jobindex ${jobindex} |
| 333 | echo recoTriggerOptions ${recoTriggerOptions} |
| 334 | echo targetDirectory ${targetDirectory} |
| 335 | echo commonOutputPath ${commonOutputPath} |
| 336 | echo doneFile ${doneFile} |
| 337 | echo runpath ${runpath} |
| 338 | echo outputDir ${outputDir} |
| 339 | echo ALICE_ROOT ${ALICE_ROOT} |
| 340 | echo PWD ${PWD} |
| fc7e1e45 |
341 | echo "########## ###########" |
| 342 | |
| 71353a5f |
343 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
344 | |
| 345 | filesCPass1=( |
| 71353a5f |
346 | "${commonOutputPath}/runCPass1.sh" |
| 347 | "${commonOutputPath}/recCPass1.C" |
| 348 | "${commonOutputPath}/recCPass1_OuterDet.C" |
| 349 | "${commonOutputPath}/runCalibTrain.C" |
| 350 | "${commonOutputPath}/QAtrain_duo.C" |
| 351 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 352 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 353 | "${commonOutputPath}/OCDB.root" |
| 354 | "${trustedQAtrainMacro}" |
| 355 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
| 356 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" |
| 357 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" |
| 358 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" |
| 359 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
360 | ) |
| 361 | |
| 362 | for file in ${filesCPass1[*]}; do |
| 363 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 364 | done |
| 365 | |
| 71353a5f |
366 | echo "this directory (${PWD}) contents:" |
| 367 | /bin/ls |
| fc7e1e45 |
368 | echo |
| 369 | |
| 71353a5f |
370 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 371 | |
| 372 | if [[ -n ${postSetUpActionCPass1} ]]; then |
| 373 | echo "running ${postSetUpActionCPass1}" |
| 374 | eval ${postSetUpActionCPass1} |
| fc7e1e45 |
375 | echo |
| 376 | fi |
| 377 | |
| 378 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 379 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
380 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 381 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
382 | else |
| 383 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 384 | fi |
| 385 | |
| 71353a5f |
386 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then |
| cb473870 |
387 | echo "cpass0 produced no calibration! exiting..." |
| 71353a5f |
388 | touch ${doneFile} |
| 389 | return 1 |
| cb473870 |
390 | fi |
| fc7e1e45 |
391 | |
| 71353a5f |
392 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory |
| 393 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB |
| 394 | #entries using a relative path, e.g. local://./OCDB |
| 395 | mkdir Barrel OuterDet |
| 396 | ln -s ../OCDB Barrel/OCDB |
| 397 | ln -s ../OCDB OuterDet/OCDB |
| 398 | |
| fc7e1e45 |
399 | #run CPass1 |
| 400 | chmod u+x runCPass1.sh |
| 71353a5f |
401 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 402 | if [[ -n ${pretend} ]]; then |
| cb473870 |
403 | touch AliESDs_Barrel.root |
| 404 | touch AliESDfriends_Barrel.root |
| fc7e1e45 |
405 | touch AliESDfriends_v1.root |
| cb473870 |
406 | touch QAresults_barrel.root |
| 407 | touch EventStat_temp_barrel.root |
| 408 | touch AODtpITS.root |
| 409 | touch AliESDs_Outer.root |
| 410 | touch AliESDfriends_Outer.root |
| 411 | touch QAresults_outer.root |
| 412 | touch EventStat_temp_outer.root |
| fc7e1e45 |
413 | touch rec.log |
| 414 | touch calib.log |
| 415 | touch qa.log |
| 416 | else |
| 71353a5f |
417 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| fc7e1e45 |
418 | fi |
| cb473870 |
419 | |
| 420 | ##handle possible crashes in QA (happens often in trunk) |
| 421 | ##rerun QA with a trusted aliroot version |
| 422 | #if [[ $(validateLog qa_barrel.log) ]]; then |
| 423 | # echo "qa_barrel.log not validated!" |
| 424 | #fi |
| 425 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then |
| 71353a5f |
426 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" |
| cb473870 |
427 | # source ${setupTrustedAliROOTenvInCurrentShell} |
| 428 | # cd Barrel |
| 429 | # rm QAresults_barrel.root |
| 430 | # rm EventStat_temp_barrel.root |
| 431 | # rm AODtpITS.root |
| 432 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root |
| 433 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root |
| 71353a5f |
434 | # if [[ -n ${trustedQAtrainMacro} ]]; then |
| 435 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" |
| cb473870 |
436 | # fi |
| 71353a5f |
437 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" |
| 438 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log |
| cb473870 |
439 | # cd ../ |
| 440 | #fi |
| 441 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root |
| 442 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root |
| 443 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . |
| 444 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . |
| 445 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root |
| 446 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root |
| 447 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . |
| 448 | |
| fc7e1e45 |
449 | #move stuff to final destination |
| 71353a5f |
450 | echo "this directory (${PWD}) contents:" |
| 451 | /bin/ls |
| 452 | echo rm -f ./${chunkName} |
| 453 | rm -f ./${chunkName} |
| fc7e1e45 |
454 | echo "cp --recursive ${runpath}/* ${outputDir}" |
| 71353a5f |
455 | cp -pf --recursive ${runpath}/* ${outputDir} |
| fc7e1e45 |
456 | echo |
| 457 | |
| 5ed7c82c |
458 | #validate CPass1 |
| 459 | cd ${outputDir} |
| 71353a5f |
460 | touch ${doneFile} |
| 461 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile} |
| 462 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} |
| 463 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile} |
| 464 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile} |
| 465 | echo "dir ${outputDir}" >> ${doneFile} |
| 466 | summarizeLogs >> ${doneFile} |
| 5ed7c82c |
467 | |
| 71353a5f |
468 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| 5de3064b |
469 | return 0 |
| fc7e1e45 |
470 | } |
| 471 | |
| 472 | |
| 473 | goMergeCPass0() |
| 474 | { |
| fc7e1e45 |
475 | # |
| 476 | # find the output files and merge them |
| 477 | # |
| 478 | |
| 71353a5f |
479 | outputDir=${1} |
| 480 | ocdbStorage=${2} |
| 481 | configFile=${3} |
| 482 | runNumber=${4} |
| 483 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly |
| 484 | shift 5 |
| 485 | parseConfig ${configFile} $@ |
| 486 | |
| 487 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 488 | doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" |
| 489 | |
| 490 | umask 0002 |
| 491 | ulimit -c unlimited |
| fc7e1e45 |
492 | |
| 71353a5f |
493 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
494 | |
| 71353a5f |
495 | #runpath=${PWD}/rundir_cpass0_Merge_${runNumber} |
| 496 | runpath=${outputDir} |
| 497 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 498 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| fc7e1e45 |
499 | |
| 71353a5f |
500 | mkdir -p ${runpath} |
| 501 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 |
| 502 | cd ${runpath} |
| fc7e1e45 |
503 | |
| 71353a5f |
504 | logOutputDir=${runpath} |
| 505 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 506 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log |
| 507 | echo "${0} $*" |
| fc7e1e45 |
508 | |
| fc7e1e45 |
509 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| fc7e1e45 |
510 | |
| 511 | echo goMergeCPass0 SETUP: |
| 71353a5f |
512 | echo runNumber=${runNumber} |
| 513 | echo outputDir=${outputDir} |
| 514 | echo ocdbStorage=${ocdbStorage} |
| 515 | echo calibrationFilesToMerge=${calibrationFilesToMerge} |
| 516 | echo mergingScript=${mergingScript} |
| 517 | echo commonOutputPath=${commonOutputPath} |
| 518 | echo runpath=${runpath} |
| fc7e1e45 |
519 | |
| 520 | # copy files in case they are not already there |
| 521 | filesMergeCPass0=( |
| 71353a5f |
522 | "${commonOutputPath}/${calibrationFilesToMerge}" |
| 523 | "${commonOutputPath}/OCDB.root" |
| 524 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 525 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
| 526 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" |
| 527 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" |
| 528 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" |
| 529 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" |
| fc7e1e45 |
530 | ) |
| 531 | for file in ${filesMergeCPass0[*]}; do |
| 532 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 533 | done |
| 534 | |
| 71353a5f |
535 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 536 | |
| 537 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
538 | |
| 539 | # |
| cb473870 |
540 | echo "PWD" |
| 71353a5f |
541 | /bin/ls |
| cb473870 |
542 | echo "PWD/.." |
| 71353a5f |
543 | /bin/ls ../ |
| cb473870 |
544 | |
| fc7e1e45 |
545 | |
| 546 | #merge calibration |
| 71353a5f |
547 | chmod u+x ${mergingScript} |
| fc7e1e45 |
548 | mkdir -p ./OCDB |
| 71353a5f |
549 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 550 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" |
| 551 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} |
| fc7e1e45 |
552 | fi |
| 553 | |
| 71353a5f |
554 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 555 | if [[ -n ${pretend} ]]; then |
| fc7e1e45 |
556 | touch CalibObjects.root |
| 557 | touch ocdb.log |
| 558 | touch merge.log |
| 71353a5f |
559 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ |
| 560 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ |
| 561 | touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root |
| 562 | touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root |
| fc7e1e45 |
563 | else |
| 71353a5f |
564 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
| fc7e1e45 |
565 | fi |
| 566 | |
| 567 | ### produce the output |
| 568 | #tar the produced OCDB for reuse |
| 71353a5f |
569 | tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB |
| fc7e1e45 |
570 | |
| 71353a5f |
571 | #produce the calib trees for expert QA (dcsTime.root) |
| 572 | goMakeLocalOCDBaccessConfig ./OCDB |
| 573 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 574 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 575 | |
| 576 | /bin/ls |
| 5ed7c82c |
577 | |
| 578 | #copy all to output dir |
| 71353a5f |
579 | cp -pf --recursive ${runpath}/* ${outputDir} |
| 5ed7c82c |
580 | |
| 71353a5f |
581 | if [[ -n ${generateMC} ]]; then |
| 582 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done |
| 583 | fi |
| 584 | |
| fc7e1e45 |
585 | #validate merging cpass0 |
| 5ed7c82c |
586 | cd ${outputDir} |
| 71353a5f |
587 | touch ${doneFile} |
| 588 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} |
| 589 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} |
| 590 | echo "dir ${outputDir}" >> ${doneFile} |
| 591 | summarizeLogs >> ${doneFile} |
| fc7e1e45 |
592 | |
| 71353a5f |
593 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| 5de3064b |
594 | return 0 |
| fc7e1e45 |
595 | } |
| 596 | |
| 597 | goMergeCPass1() |
| 598 | { |
| fc7e1e45 |
599 | # |
| 600 | # find the output files and merge them |
| 601 | # |
| 602 | |
| 71353a5f |
603 | outputDir=${1} |
| 604 | ocdbStorage=${2} |
| 605 | configFile=${3} |
| 606 | runNumber=${4} |
| 607 | calibrationFilesToMerge=${5} |
| 608 | qaFilesToMerge=${6} |
| 609 | shift 6 |
| 610 | parseConfig ${configFile} $@ |
| 611 | |
| 612 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 613 | doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" |
| 614 | |
| 615 | umask 0002 |
| 616 | ulimit -c unlimited |
| fc7e1e45 |
617 | |
| 618 | #clean up first: |
| 71353a5f |
619 | rm -f ${outputDir}/*.log |
| 620 | rm -f ${outputDir}/*.root |
| 621 | rm -f ${outputDir}/*done |
| 622 | |
| 623 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 624 | |
| 625 | #runpath=${PWD}/rundir_cpass1_Merge_${runNumber} |
| 626 | runpath=${outputDir} |
| 627 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 628 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) |
| 629 | |
| 630 | mkdir -p ${runpath} |
| 631 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 |
| 632 | cd ${runpath} |
| 633 | |
| 634 | logOutputDir=${runpath} |
| 635 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 636 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log |
| 637 | echo "${0} $*" |
| 638 | |
| 639 | calibrationOutputFileName='AliESDfriends_v1.root' |
| 640 | qaOutputFileName='QAresults*.root' |
| fc7e1e45 |
641 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| cb473870 |
642 | #important to have the string "Stage.txt" in the filename to trigger the merging |
| 643 | #it has to be a list of directories containing the files |
| fc7e1e45 |
644 | qaMergedOutputFileName="QAresults_merged.root" |
| 645 | |
| 646 | echo goMergeCPass1 SETUP: |
| 71353a5f |
647 | echo runNumber=${runNumber} |
| 648 | echo outputDir=${outputDir} |
| 649 | echo ocdbStorage=${ocdbStorage} |
| 650 | echo calibrationFilesToMerge=$calibrationFilesToMerge |
| 651 | echo qaFilesToMerge=$qaFilesToMerge |
| 652 | echo calibrationOutputFileName=${calibrationOutputFileName} |
| 653 | echo mergingScript=${mergingScript} |
| fc7e1e45 |
654 | |
| 655 | # copy files in case they are not already there |
| 656 | filesMergeCPass1=( |
| 71353a5f |
657 | "${commonOutputPath}/${calibrationFilesToMerge}" |
| 658 | "${commonOutputPath}/${qaFilesToMerge}" |
| 659 | "${commonOutputPath}/OCDB.root" |
| 660 | "${commonOutputPath}/localOCDBaccessConfig.C" |
| 661 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 662 | "${commonOutputPath}/QAtrain_duo.C" |
| 663 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
| 664 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" |
| 665 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" |
| 666 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" |
| 667 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" |
| 668 | "${trustedQAtrainMacro}" |
| 669 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
670 | ) |
| 671 | for file in ${filesMergeCPass1[*]}; do |
| 672 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 673 | done |
| 674 | |
| 71353a5f |
675 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 676 | |
| cb473870 |
677 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 678 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
679 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 680 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
681 | else |
| 682 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 683 | fi |
| 684 | |
| 71353a5f |
685 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
686 | |
| 687 | # |
| 71353a5f |
688 | /bin/ls |
| fc7e1e45 |
689 | |
| 690 | #merge calibration |
| 71353a5f |
691 | chmod u+x ${mergingScript} |
| fc7e1e45 |
692 | mkdir -p OCDB |
| 71353a5f |
693 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 694 | echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}" |
| 695 | /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge} |
| fc7e1e45 |
696 | fi |
| 697 | |
| 71353a5f |
698 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 699 | if [[ -n ${pretend} ]]; then |
| fc7e1e45 |
700 | touch CalibObjects.root |
| 701 | touch ocdb.log |
| 702 | touch merge.log |
| 71353a5f |
703 | touch cpass1.localOCDB.${runNumber}.tgz |
| fc7e1e45 |
704 | else |
| 71353a5f |
705 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
| fc7e1e45 |
706 | fi |
| 707 | |
| 71353a5f |
708 | tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB |
| fc7e1e45 |
709 | |
| 710 | #merge QA |
| 5ed7c82c |
711 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF |
| 712 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF |
| 713 | |
| 71353a5f |
714 | if [[ ! -f ${qaFilesToMerge} ]]; then |
| cb473870 |
715 | #find the files, but only store the directories (QAtrain_duo.C requires this) |
| 71353a5f |
716 | echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" |
| 717 | eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} |
| fc7e1e45 |
718 | fi |
| 719 | |
| cb473870 |
720 | #do the merge |
| 71353a5f |
721 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 722 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" |
| 723 | if [[ -n ${pretend} ]]; then |
| 724 | touch ${qaMergedOutputFileName} |
| fc7e1e45 |
725 | touch merge.log |
| cb473870 |
726 | touch trending.root |
| fc7e1e45 |
727 | else |
| 71353a5f |
728 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 729 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log |
| 730 | mv QAresults_barrel.root ${qaMergedOutputFileName} |
| cb473870 |
731 | mv trending_barrel.root trending.root |
| fc7e1e45 |
732 | fi |
| cb473870 |
733 | |
| 71353a5f |
734 | #produce the calib trees for expert QA |
| 735 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 736 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 737 | |
| 738 | /bin/ls |
| 5ed7c82c |
739 | |
| 740 | #copy all to output dir |
| 71353a5f |
741 | cp -pf --recursive ${runpath}/* ${outputDir} |
| 5ed7c82c |
742 | |
| fc7e1e45 |
743 | #validate merge cpass1 |
| 5ed7c82c |
744 | cd ${outputDir} |
| 71353a5f |
745 | touch ${doneFile} |
| 746 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} |
| 747 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile} |
| 748 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile} |
| 749 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} |
| 750 | echo "dir ${outputDir}" >> ${doneFile} |
| 751 | summarizeLogs >> ${doneFile} |
| 752 | |
| 753 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| 5de3064b |
754 | return 0 |
| cb473870 |
755 | } |
| 756 | |
| 757 | goMerge() |
| 758 | { |
| 759 | #generic root merge using CPass1 merge.C script |
| 71353a5f |
760 | inputList=${1} |
| 761 | outputFile=${2} |
| cb473870 |
762 | configFile=${3-"becnhmark.config"} |
| 71353a5f |
763 | shift 3 |
| 764 | parseConfig ${configFile} $@ |
| fc7e1e45 |
765 | |
| 71353a5f |
766 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 |
| 767 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 |
| 768 | umask 0002 |
| 769 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 770 | rm -f ${outputFile} |
| 771 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log |
| 5de3064b |
772 | return 0 |
| 5dfac970 |
773 | } |
| fc7e1e45 |
774 | |
| cb473870 |
775 | goSubmitMakeflow() |
| 5dfac970 |
776 | { |
| cb473870 |
777 | #run |
| 71353a5f |
778 | productionID=${1} |
| 779 | inputFileList=${2} |
| 780 | configFile=${3} |
| 781 | shift 3 |
| 782 | extraOpts="$@" |
| 783 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
784 | |
| 5dfac970 |
785 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 786 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
787 | |
| 71353a5f |
788 | #create the directopry for the metadata |
| 789 | mkdir meta |
| 790 | |
| 791 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then |
| cb473870 |
792 | echo "setting the makflow path from the config: " |
| 793 | echo " export PATH=${makeflowPath}:${PATH}" |
| 794 | export PATH=${makeflowPath}:${PATH} |
| 795 | fi |
| 796 | |
| 797 | #submit - use makeflow if available, fall back to old stuff when makeflow not there |
| 798 | if which makeflow; then |
| cb473870 |
799 | |
| 71353a5f |
800 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow |
| 801 | |
| cb473870 |
802 | makeflow ${makeflowOptions} benchmark.makeflow |
| 803 | cd ../ |
| 804 | else |
| 805 | echo "no makeflow!" |
| 806 | fi |
| 5de3064b |
807 | return 0 |
| 5dfac970 |
808 | } |
| fc7e1e45 |
809 | |
| 5dfac970 |
810 | goGenerateMakeflow() |
| 811 | { |
| 812 | #generate the makeflow file |
| 71353a5f |
813 | productionID=${1} |
| 814 | inputFileList=${2} |
| 815 | configFile=${3} |
| 816 | shift 3 |
| 817 | extraOpts="$@" |
| 818 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
819 | |
| 5dfac970 |
820 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 821 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
822 | |
| 5dfac970 |
823 | commonOutputPath=${baseOutputDirectory}/${productionID} |
| fc7e1e45 |
824 | |
| 5dfac970 |
825 | #these files will be made a dependency - will be copied to the working dir of the jobs |
| 826 | declare -a copyFiles |
| 827 | inputFiles=( |
| 828 | "OCDB.root" |
| 829 | "localOCDBaccessConfig.C" |
| cb473870 |
830 | "QAtrain_duo.C" |
| 831 | "runCPass1.sh" |
| 832 | "recCPass1.C" |
| 833 | "recCPass1_OuterDet.C" |
| 834 | "runCalibTrain.C" |
| 835 | "runCPass0.sh" |
| 836 | "recCPass0.C" |
| 5dfac970 |
837 | ) |
| 838 | for file in ${inputFiles[*]}; do |
| 839 | [[ -f ${file} ]] && copyFiles+=("${file}") |
| 840 | done |
| 841 | |
| 842 | #create the makeflow file |
| cb473870 |
843 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" |
| 5dfac970 |
844 | declare -a arr_cpass1_final |
| 71353a5f |
845 | listOfRuns[0]=${runNumber} |
| 846 | [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq)) |
| 5dfac970 |
847 | runindex=0 |
| fc7e1e45 |
848 | for runNumber in ${listOfRuns[*]}; do |
| 71353a5f |
849 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
850 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| fc7e1e45 |
851 | |
| cb473870 |
852 | unset arr_cpass0_outputs_onerun |
| 853 | unset arr_cpass1_outputs_onerun |
| 854 | declare -a arr_cpass0_outputs_onerun |
| 855 | declare -a arr_cpass1_outputs_onerun |
| fc7e1e45 |
856 | |
| cb473870 |
857 | jobindex=0 |
| 5dfac970 |
858 | while read inputFile; do |
| 859 | currentDefaultOCDB=${defaultOCDB} |
| cb473870 |
860 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 861 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
862 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) |
| cb473870 |
863 | fi |
| fc7e1e45 |
864 | |
| 5dfac970 |
865 | #CPass0 |
| 71353a5f |
866 | arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done" |
| 867 | echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}" |
| 868 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}" |
| fc7e1e45 |
869 | echo |
| 5dfac970 |
870 | |
| 871 | #CPass1 |
| 71353a5f |
872 | arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done" |
| 873 | echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" |
| 874 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}" |
| fc7e1e45 |
875 | echo |
| 5dfac970 |
876 | ((jobindex++)) |
| fc7e1e45 |
877 | |
| 71353a5f |
878 | done< <(grep "/000${runNumber}/" ${inputFileList}) |
| cb473870 |
879 | |
| 5dfac970 |
880 | #CPass0 list of Calib files to merge |
| 71353a5f |
881 | arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list" |
| 882 | echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}" |
| 883 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}" |
| 884 | echo |
| 885 | |
| 886 | #CPass0 merging |
| 887 | arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done" |
| 888 | echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}" |
| 889 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}" |
| 5dfac970 |
890 | echo |
| fc7e1e45 |
891 | |
| 5dfac970 |
892 | #CPass1 list of Calib/QA files to merge |
| cb473870 |
893 | # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger |
| 8d6a0a43 |
894 | # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid |
| 895 | # this corresponds to the last merging stage) |
| 71353a5f |
896 | arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list" |
| 897 | echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 898 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};" |
| cb473870 |
899 | echo |
| 71353a5f |
900 | arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" |
| 901 | echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 902 | echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}" |
| 5dfac970 |
903 | echo |
| fc7e1e45 |
904 | |
| 71353a5f |
905 | #CPass1 merging |
| 906 | arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done" |
| 907 | echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}" |
| 908 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}" |
| 5dfac970 |
909 | echo |
| fc7e1e45 |
910 | |
| 71353a5f |
911 | arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list" |
| 912 | echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}" |
| 913 | echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}" |
| cb473870 |
914 | |
| 71353a5f |
915 | if [ ${runESDfiltering} -eq 1 ]; then |
| 916 | arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done" |
| 917 | echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}" |
| 918 | echo " ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}" |
| cb473870 |
919 | fi |
| 920 | |
| 5dfac970 |
921 | ((runindex++)) |
| 922 | done |
| fc7e1e45 |
923 | |
| 5dfac970 |
924 | #Summary |
| 71353a5f |
925 | echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}" |
| 926 | echo " LOCAL ./benchmark.sh MakeSummary ${configFile}" |
| cb473870 |
927 | echo |
| 928 | |
| 5de3064b |
929 | return 0 |
| cb473870 |
930 | } |
| 931 | |
| 932 | goPrintValues() |
| 933 | { |
| 934 | #print the values given the key from any number of files (store in output file on second argument) |
| 71353a5f |
935 | if [[ $# -lt 3 ]]; then |
| 936 | echo "goPrintValues key outputFile inputFiles" |
| 937 | echo "if outputFile is \"-\" dont redirect to a file" |
| 938 | return |
| 939 | fi |
| 940 | key=${1} |
| 941 | outputFile=${2} |
| 942 | [[ ${outputFile} =~ "-" ]] && outputFile="" |
| cb473870 |
943 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk |
| 71353a5f |
944 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} |
| 5de3064b |
945 | return 0 |
| 5dfac970 |
946 | } |
| fc7e1e45 |
947 | |
| 5dfac970 |
948 | goCreateQAplots() |
| 949 | { |
| 950 | umask 0002 |
| 71353a5f |
951 | mergedQAfileList=${1} |
| 952 | productionID=${2} |
| 953 | outputDir=${3} |
| 954 | configFile=${4} |
| 955 | shift 4 |
| 956 | parseConfig ${configFile} ${@} |
| 957 | |
| 958 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
959 | |
| 71353a5f |
960 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} |
| 961 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log |
| 962 | echo "${0} $*" |
| 963 | |
| 964 | olddir=${PWD} |
| 965 | mkdir -p ${outputDir} |
| 966 | cd ${outputDir} |
| 967 | [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1 |
| fc7e1e45 |
968 | |
| 71353a5f |
969 | echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList} |
| 970 | ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList} |
| fc7e1e45 |
971 | |
| 71353a5f |
972 | cd ${olddir} |
| 5de3064b |
973 | return 0 |
| 5dfac970 |
974 | } |
| fc7e1e45 |
975 | |
| 5dfac970 |
976 | goTest() |
| 977 | { |
| 978 | umask 0002 |
| 979 | exec 2>&1 |
| 980 | exec > >(tee test.log) |
| 981 | echo "$@" |
| 982 | echo something |
| 5de3064b |
983 | return 0 |
| 5dfac970 |
984 | } |
| fc7e1e45 |
985 | |
| 5dfac970 |
986 | alirootInfo() |
| 987 | { |
| 5dfac970 |
988 | umask 0002 |
| 71353a5f |
989 | # save aliroot repository info |
| 990 | [[ -z "${ALICE_ROOT}" ]] && return 1 |
| 991 | |
| 992 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" |
| 993 | echo "\${ROOTSYS}=${ROOTSYS}" |
| 994 | echo "\${PATH}=${PATH}" |
| 995 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" |
| 996 | |
| 997 | pushd ${PWD} |
| 998 | cd ${ALICE_ROOT} |
| 5dfac970 |
999 | echo |
| 71353a5f |
1000 | |
| 1001 | currentBranch=$(git rev-parse --abbrev-ref HEAD) |
| 1002 | git status |
| 5dfac970 |
1003 | echo "" |
| 1004 | echo "" |
| 71353a5f |
1005 | git diff ${currentBranch} |
| 1006 | popd |
| 5de3064b |
1007 | return 0 |
| 5dfac970 |
1008 | } |
| fc7e1e45 |
1009 | |
| 5dfac970 |
1010 | setYear() |
| 1011 | { |
| 1012 | #set the year |
| 71353a5f |
1013 | # ${1} - year to be set |
| 1014 | # ${2} - where to set the year |
| 1015 | year1=$(guessYear ${1}) |
| 1016 | year2=$(guessYear ${2}) |
| 1017 | local path=${2} |
| 1018 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} |
| 1019 | echo ${path} |
| 5de3064b |
1020 | return 0 |
| 5dfac970 |
1021 | } |
| fc7e1e45 |
1022 | |
| 5dfac970 |
1023 | guessPeriod() |
| 1024 | { |
| 1025 | #guess the period from the path, pick the rightmost one |
| 1026 | local IFS="/" |
| 71353a5f |
1027 | declare -a path=( ${1} ) |
| 5dfac970 |
1028 | local dirDepth=${#path[*]} |
| 1029 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1030 | local field=${path[${x}]} |
| 71353a5f |
1031 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break |
| 5dfac970 |
1032 | done |
| 71353a5f |
1033 | echo ${period} |
| 5de3064b |
1034 | return 0 |
| 5dfac970 |
1035 | } |
| fc7e1e45 |
1036 | |
| 5dfac970 |
1037 | guessYear() |
| 1038 | { |
| 1039 | #guess the year from the path, pick the rightmost one |
| 1040 | local IFS="/" |
| 71353a5f |
1041 | declare -a path=( ${1} ) |
| 5dfac970 |
1042 | local dirDepth=${#path[*]} |
| 1043 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1044 | local field=${path[${x}]} |
| 71353a5f |
1045 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break |
| 5dfac970 |
1046 | done |
| 71353a5f |
1047 | echo ${year} |
| 5de3064b |
1048 | return 0 |
| 5dfac970 |
1049 | } |
| fc7e1e45 |
1050 | |
| 5dfac970 |
1051 | guessRunNumber() |
| 1052 | { |
| 1053 | #guess the run number from the path, pick the rightmost one |
| 71353a5f |
1054 | #works for /path/foo/000123456/bar/... |
| 1055 | #and /path/foo.run123456.bar |
| 1056 | local IFS="/." |
| 1057 | declare -a path=( ${1} ) |
| 5dfac970 |
1058 | local dirDepth=${#path[*]} |
| 1059 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1060 | local field=${path[${x}]} |
| 71353a5f |
1061 | field=${field/run/000} |
| 5dfac970 |
1062 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break |
| 1063 | done |
| 71353a5f |
1064 | echo ${runNumber} |
| 5de3064b |
1065 | return 0 |
| 5dfac970 |
1066 | } |
| fc7e1e45 |
1067 | |
| cb473870 |
1068 | validateLog() |
| 1069 | { |
| 71353a5f |
1070 | log=${1} |
| cb473870 |
1071 | errorConditions=( |
| 71353a5f |
1072 | 'There was a crash' |
| 1073 | 'floating' |
| 1074 | 'error while loading shared libraries' |
| 1075 | 'std::bad_alloc' |
| 1076 | 's_err_syswatch_' |
| 1077 | 'Thread [0-9]* (Thread' |
| 1078 | 'AliFatal' |
| 1079 | 'core dumped' |
| 1080 | '\.C.*error:.*\.h: No such file' |
| 1081 | ) |
| 1082 | |
| 1083 | warningConditions=( |
| 1084 | 'This is serious !' |
| 1085 | 'rocVoltage out of range:' |
| cb473870 |
1086 | ) |
| 1087 | |
| 71353a5f |
1088 | local logstatus=0 |
| 1089 | local errorSummary="" |
| 1090 | local warningSummary="" |
| 1091 | |
| 1092 | for ((i=0; i<${#errorConditions[@]};i++)); do |
| 1093 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) |
| 1094 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1095 | errorSummary+=${tmp} |
| 1096 | done |
| 1097 | |
| 1098 | for ((i=0; i<${#warningConditions[@]};i++)); do |
| 1099 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) |
| 1100 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1101 | warningSummary+=${tmp} |
| 1102 | done |
| 1103 | |
| 1104 | if [[ -n ${errorSummary} ]]; then |
| 1105 | echo "${errorSummary}" |
| 1106 | return 1 |
| 1107 | fi |
| cb473870 |
1108 | |
| 71353a5f |
1109 | if [[ -n ${warningSummary} ]]; then |
| 1110 | echo "${warningSummary}" |
| 1111 | return 2 |
| 1112 | fi |
| 1113 | |
| 1114 | return 0 |
| cb473870 |
1115 | } |
| 1116 | |
| 5dfac970 |
1117 | summarizeLogs() |
| 1118 | { |
| 1119 | #print a summary of logs |
| 1120 | logFiles=( |
| 1121 | "*.log" |
| cb473870 |
1122 | "stdout" |
| 1123 | "stderr" |
| 5dfac970 |
1124 | ) |
| fc7e1e45 |
1125 | |
| cb473870 |
1126 | #check logs |
| 71353a5f |
1127 | local logstatus=0 |
| 5dfac970 |
1128 | for log in ${logFiles[*]}; do |
| 71353a5f |
1129 | finallog=${PWD%/}/${log} |
| 1130 | [[ ! -f ${log} ]] && continue |
| 1131 | errorSummary=$(validateLog ${log}) |
| 1132 | validationStatus=$? |
| 1133 | if [[ ${validationStatus} -eq 0 ]]; then |
| 5dfac970 |
1134 | #in pretend mode randomly report an error in rec.log some cases |
| 71353a5f |
1135 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then |
| 1136 | [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" |
| fc7e1e45 |
1137 | else |
| 71353a5f |
1138 | echo "${finallog} OK" |
| fc7e1e45 |
1139 | fi |
| 71353a5f |
1140 | elif [[ ${validationStatus} -eq 1 ]]; then |
| 1141 | echo "${finallog} BAD ${errorSummary}" |
| 1142 | elif [[ ${validationStatus} -eq 2 ]]; then |
| 1143 | echo "${finallog} OK MWAH ${errorSummary}" |
| 5dfac970 |
1144 | fi |
| 1145 | done |
| cb473870 |
1146 | |
| 1147 | #report core files |
| 71353a5f |
1148 | while read x; do |
| 1149 | echo ${x} |
| 1150 | chmod 644 ${x} |
| 1151 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log |
| 1152 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) |
| cb473870 |
1153 | |
| 71353a5f |
1154 | return ${logstatus} |
| 5dfac970 |
1155 | } |
| fc7e1e45 |
1156 | |
| 5dfac970 |
1157 | spitOutLocalOCDBaccessConfig() |
| 1158 | { |
| 1159 | umask 0002 |
| 71353a5f |
1160 | #find ${1} -name "*root" | \ |
| 1161 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ |
| 5dfac970 |
1162 | while read line |
| 1163 | do |
| 71353a5f |
1164 | local tmp=${line#${1}} |
| 5dfac970 |
1165 | echo ${tmp%/*} | \ |
| 71353a5f |
1166 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' |
| 5dfac970 |
1167 | done |
| 5de3064b |
1168 | return 0 |
| 5dfac970 |
1169 | } |
| fc7e1e45 |
1170 | |
| 71353a5f |
1171 | goMakeLocalOCDBaccessConfig() |
| 5dfac970 |
1172 | { |
| 1173 | umask 0002 |
| 1174 | # make a script that sets the specific storages form all the root files produced by CPass0 |
| 71353a5f |
1175 | local localOCDBpathCPass0=${1} |
| 1176 | local OCDBpathPrefix=${2} |
| 1177 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." |
| 1178 | |
| 1179 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then |
| 1180 | tar xzf ${localOCDBpathCPass0} |
| 1181 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" |
| 5dfac970 |
1182 | fi |
| fc7e1e45 |
1183 | |
| 5dfac970 |
1184 | echo |
| 1185 | echo creating the specific storage script |
| 1186 | echo localOCDBaccessConfig.C |
| 71353a5f |
1187 | echo based on OCDB: ${localOCDBaccessConfig} |
| 5dfac970 |
1188 | echo |
| fc7e1e45 |
1189 | |
| 5dfac970 |
1190 | local tempLocalOCDB="" |
| 1191 | if [[ -f localOCDBaccessConfig.C ]]; then |
| 1192 | tempLocalOCDB=$(mktemp) |
| 71353a5f |
1193 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" |
| 1194 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} |
| 5dfac970 |
1195 | fi |
| fc7e1e45 |
1196 | |
| 5dfac970 |
1197 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
| 1198 | echo "{" >> localOCDBaccessConfig.C |
| 1199 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C |
| 71353a5f |
1200 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C |
| 1201 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C |
| 5dfac970 |
1202 | echo "}" >> localOCDBaccessConfig.C |
| fc7e1e45 |
1203 | |
| 71353a5f |
1204 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} |
| fc7e1e45 |
1205 | |
| 5dfac970 |
1206 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
| 1207 | echo |
| 1208 | echo "!!!!!!! CPass0 produced no OCDB entries" |
| 1209 | return 1 |
| 1210 | fi |
| 5de3064b |
1211 | return 0 |
| 5dfac970 |
1212 | } |
| fc7e1e45 |
1213 | |
| cb473870 |
1214 | goMakeFilteredTrees() |
| 1215 | { |
| 71353a5f |
1216 | outputDir=${1} |
| 1217 | runNumber=${2} |
| cb473870 |
1218 | #get path to input list |
| 71353a5f |
1219 | inputListfiles=${3} |
| cb473870 |
1220 | #get scale number for tracks |
| 71353a5f |
1221 | filterT=${4} |
| cb473870 |
1222 | #get scale number for V0s |
| 71353a5f |
1223 | filterV=${5} |
| cb473870 |
1224 | #get OCDB path (optional) |
| 1225 | OCDBpath=${6} |
| 1226 | #get max number of files |
| 1227 | maxFiles=${7-"1000000"} |
| 1228 | #get offset of first file |
| 1229 | offsetFile=${8-"0"} |
| 1230 | #get max number of events |
| 1231 | maxEvents=${9-"30000000"} |
| 1232 | #get offset of first event |
| 1233 | offsetEvent=${10-"0"} |
| 1234 | configFile=${11-"benchmark.config"} |
| 1235 | esdFileName=${12-"AliESDs_Barrel.root"} |
| 71353a5f |
1236 | shift 12 |
| 1237 | parseConfig ${configFile} $@ |
| cb473870 |
1238 | |
| 1239 | commonOutputPath=${PWD} |
| 71353a5f |
1240 | doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done |
| 1241 | |
| cb473870 |
1242 | runpath=${commonOutputPath}/rundir_filtering_run${runNumber} |
| 1243 | |
| 71353a5f |
1244 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 1245 | |
| cb473870 |
1246 | mkdir -p ${outputDir} |
| 1247 | mkdir -p ${runpath} |
| 1248 | |
| 1249 | cd ${runpath} |
| 1250 | |
| 1251 | cat > filtering.log << EOF |
| 1252 | goMakeFilteredTrees config: |
| 71353a5f |
1253 | runpath=${runpath} |
| 1254 | outputDir=${outputDir} |
| 1255 | commonOutputPath=${commonOutputPath} |
| 1256 | ALICE_ROOT=${ALICE_ROOT} |
| 1257 | PATH=${PATH} |
| 1258 | offsetEvent=$offsetEvent |
| 1259 | configFile=$configFile |
| 1260 | esdFileName=$esdFileName |
| cb473870 |
1261 | EOF |
| 1262 | |
| 1263 | if [[ -z ${pretend} ]];then |
| 71353a5f |
1264 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log |
| cb473870 |
1265 | else |
| 71353a5f |
1266 | touch filtering.log FilterEvents_Trees.root |
| cb473870 |
1267 | fi |
| 1268 | pwd |
| 71353a5f |
1269 | /bin/ls |
| 1270 | touch ${doneFile} |
| 1271 | summarizeLogs >> ${doneFile} |
| cb473870 |
1272 | echo mv -f * ${outputDir} |
| 1273 | mv -f * ${outputDir} |
| 71353a5f |
1274 | [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} |
| cb473870 |
1275 | cd ${commonOutputPath} |
| 71353a5f |
1276 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| 5de3064b |
1277 | return 0 |
| cb473870 |
1278 | } |
| 1279 | |
| 1280 | submit() |
| 1281 | { |
| 1282 | umask 0002 |
| 71353a5f |
1283 | [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1 |
| 1284 | JobID=${1} |
| 1285 | startID=${2} |
| 1286 | endID=${3} |
| 1287 | waitForJOBID=${4} |
| 1288 | command=${5} |
| 1289 | commandArgs=${6} |
| cb473870 |
1290 | |
| 71353a5f |
1291 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 |
| cb473870 |
1292 | |
| 1293 | newFarm=$(which qsub|grep "^/usr/bin/qsub") |
| 1294 | |
| 1295 | batchSystem="SGE" |
| 1296 | |
| 71353a5f |
1297 | if [[ -z "${newFarm}" ]] |
| cb473870 |
1298 | then |
| 1299 | #old LSF |
| 1300 | # submit it (as job array) |
| 71353a5f |
1301 | nFiles=$(( ${endID}-${startID}+1 )) |
| 1302 | while [ ${startID} -le ${nFiles} ] ; do |
| 1303 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then |
| 1304 | endID=$(expr ${startID} + 999) |
| cb473870 |
1305 | else |
| 71353a5f |
1306 | endID=${nFiles} |
| cb473870 |
1307 | fi |
| 1308 | if [[ ${waitForJOBID} -eq 0 ]]; then |
| 71353a5f |
1309 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1310 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1311 | else |
| 71353a5f |
1312 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1313 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1314 | fi |
| 71353a5f |
1315 | startID=$(expr ${endID} + 1) |
| cb473870 |
1316 | done |
| 1317 | else |
| 1318 | #new SGE farm |
| 1319 | if [[ ${waitForJOBID} =~ "000" ]]; then |
| 71353a5f |
1320 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| 1321 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| cb473870 |
1322 | else |
| 71353a5f |
1323 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| 1324 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs} |
| cb473870 |
1325 | fi |
| 1326 | fi |
| 5de3064b |
1327 | return 0 |
| cb473870 |
1328 | } |
| 1329 | |
| 1330 | goSubmitBatch() |
| 5dfac970 |
1331 | { |
| 71353a5f |
1332 | if [[ $# -lt 3 ]]; then |
| 1333 | echo "minimal use:" |
| 1334 | echo " ${0} submit fileList productionID configFile" |
| 1335 | return 0 |
| 1336 | fi |
| 1337 | |
| 1338 | productionID=${1} |
| 1339 | inputList=${2} |
| 1340 | configFile=${3:-"benchmark.config"} |
| 1341 | configFile=$(readlink -f ${configFile}) |
| 1342 | shift 3 |
| 1343 | extraOpts="$@" |
| 1344 | parseConfig ${configFile} ${extraOpts} |
| fc7e1e45 |
1345 | |
| 5dfac970 |
1346 | #redirect all output to submit.log |
| 1347 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" |
| 1348 | exec 7>&1 |
| 1349 | exec 1>submit_${productionID//"/"/_}.log 2>&1 |
| fc7e1e45 |
1350 | |
| 5dfac970 |
1351 | umask 0002 |
| 71353a5f |
1352 | echo ${0}" submit $*" |
| 1353 | if [[ -z "${inputList}" || -z "${productionID}" ]] |
| 5dfac970 |
1354 | then |
| fc7e1e45 |
1355 | echo |
| 71353a5f |
1356 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" |
| fc7e1e45 |
1357 | echo |
| 71353a5f |
1358 | return |
| fc7e1e45 |
1359 | fi |
| fc7e1e45 |
1360 | |
| 5dfac970 |
1361 | # check if config file is there |
| 71353a5f |
1362 | if [ ! -f ${configFile} ]; then |
| 1363 | echo "ERROR! Config File '${configFile}' not found" >&2 |
| 1364 | return |
| 5dfac970 |
1365 | else |
| 71353a5f |
1366 | echo "Using Config File: '${configFile}'" |
| fc7e1e45 |
1367 | fi |
| 1368 | |
| 5dfac970 |
1369 | # and print the configuration |
| 71353a5f |
1370 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
1371 | |
| 71353a5f |
1372 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there |
| 1373 | self=$(readlink -f "${0}") |
| 1374 | configPath=$(dirname ${configFile}) |
| 5dfac970 |
1375 | export commonOutputPath=${baseOutputDirectory}/${productionID} |
| 71353a5f |
1376 | |
| cb473870 |
1377 | mkdir -p ${commonOutputPath} |
| 71353a5f |
1378 | mkdir -p ${commonOutputPath}/logs |
| 1379 | mkdir -p ${commonOutputPath}/meta |
| 1380 | |
| 1381 | cp ${self} ${commonOutputPath} |
| 1382 | cp ${configFile} ${commonOutputPath} |
| 1383 | cp ${inputList} ${commonOutputPath} |
| cb473870 |
1384 | self=${commonOutputPath}/${self##*/} |
| 71353a5f |
1385 | chmod u+x ${self} |
| cb473870 |
1386 | configFile=${commonOutputPath}/${configFile##*/} |
| 1387 | inputList=${commonOutputPath}/${inputList##*/} |
| fc7e1e45 |
1388 | |
| 5dfac970 |
1389 | #convert to absolut pathnames |
| 71353a5f |
1390 | inputList=$(readlink -f "${inputList}") |
| 5dfac970 |
1391 | #make list of runs |
| 71353a5f |
1392 | if [[ -z ${runNumber} ]]; then |
| 1393 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) |
| 5dfac970 |
1394 | else |
| 71353a5f |
1395 | listOfRuns=${runNumber} |
| 5dfac970 |
1396 | fi |
| fc7e1e45 |
1397 | |
| 71353a5f |
1398 | alirootSource=$(readlink -f "${alirootSource}") |
| fc7e1e45 |
1399 | |
| 5dfac970 |
1400 | echo "" |
| 1401 | echo "### BEGIN CONFIGURATION ###" |
| 1402 | echo "" |
| 1403 | echo "GENERAL:" |
| 1404 | echo "" |
| 71353a5f |
1405 | echo " productionID: ${productionID}" |
| 1406 | echo " batchCommand: ${batchCommand}" |
| 1407 | echo " batchFlags: ${batchFlags}" |
| 1408 | echo " alirootSource: ${alirootSource}" |
| 1409 | echo " ALICE_ROOT: ${ALICE_ROOT}" |
| 1410 | echo " ALIROOT_RELEASE: ${ALICE_RELEASE}" |
| 1411 | echo " inputList: ${inputList}" |
| 1412 | echo " configPath: ${configPath}" |
| 1413 | echo " commonOutputPath: ${commonOutputPath}" |
| 1414 | echo " defaultOCDB: ${defaultOCDB}" |
| 1415 | echo " autoOCDB: ${autoOCDB}" |
| 1416 | echo " recoTriggerOptions: ${recoTriggerOptions}" |
| 5dfac970 |
1417 | echo " runs:" |
| 1418 | echo " ${listOfRuns[*]}" |
| 1419 | echo "" |
| 1420 | echo "THE TRAIN WILL RUN:" |
| fc7e1e45 |
1421 | |
| 71353a5f |
1422 | if [ ${runCPass0reco} -eq 1 ]; then |
| 5dfac970 |
1423 | echo " Pass0 - Recontruction" |
| 1424 | fi |
| fc7e1e45 |
1425 | |
| 71353a5f |
1426 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1427 | echo " Pass0 - merging and OCDB export" |
| 1428 | fi |
| fc7e1e45 |
1429 | |
| 71353a5f |
1430 | if [ ${runCPass1reco} -eq 1 ]; then |
| 5dfac970 |
1431 | echo " Pass1 - Recontruction" |
| 1432 | fi |
| 71353a5f |
1433 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1434 | echo " Pass1 - merging and OCDB export" |
| 1435 | fi |
| fc7e1e45 |
1436 | |
| 5dfac970 |
1437 | echo "" |
| 1438 | echo "LIMITS:" |
| 71353a5f |
1439 | echo " max. Events/Chunk: ${nEvents}" |
| 1440 | echo " max. Number of Chunks per Run: ${nMaxChunks}" |
| 5dfac970 |
1441 | echo "" |
| 1442 | echo "### END CONFIGURATION ###" |
| 1443 | echo "" |
| fc7e1e45 |
1444 | |
| 36b5de1a |
1445 | |
| 5dfac970 |
1446 | # check if input file is there |
| 71353a5f |
1447 | if [ ! -f ${inputList} ]; then |
| 1448 | echo "ERROR! Input List '${inputList}' not found" >&2 |
| 1449 | return |
| 5dfac970 |
1450 | fi |
| 1451 | |
| 1452 | # define jobid (for dependent jobs) |
| cb473870 |
1453 | date=$(date +%Y_%m_%d_%H%M%S) |
| 5dfac970 |
1454 | #for each run we submit one jobarray: |
| fc7e1e45 |
1455 | for runNumber in ${listOfRuns[*]}; do |
| cb473870 |
1456 | |
| 71353a5f |
1457 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
1458 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| cb473870 |
1459 | |
| 1460 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" |
| 1461 | JOBID1="p0_${JOBpostfix}" |
| 1462 | JOBID1wait="w0_${JOBpostfix}" |
| 1463 | JOBID2="m0_${JOBpostfix}" |
| 1464 | JOBID2wait="wm0_${JOBpostfix}" |
| 1465 | JOBID3="op0_${JOBpostfix}" |
| 1466 | JOBID3wait="wop0_${JOBpostfix}" |
| 1467 | JOBID4="p1_${JOBpostfix}" |
| 1468 | JOBID4wait="w1_${JOBpostfix}" |
| 1469 | JOBID5="m1_${JOBpostfix}" |
| 1470 | JOBID5wait="wm1_${JOBpostfix}" |
| 1471 | JOBID6="s1_${JOBpostfix}" |
| 1472 | JOBID6wait="ws1_${JOBpostfix}" |
| 1473 | JOBID7="QA_${JOBpostfix}" |
| 1474 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" |
| 1475 | JOBfilterESDcpass1="fp1_${JOBpostfix}" |
| 1476 | LASTJOB="000" |
| 1477 | |
| 71353a5f |
1478 | oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList}) |
| cb473870 |
1479 | |
| 1480 | currentDefaultOCDB=${defaultOCDB} |
| 1481 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 1482 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
1483 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) |
| cb473870 |
1484 | fi |
| 1485 | |
| 71353a5f |
1486 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" |
| 1487 | |
| 1488 | ############################################################################### |
| 1489 | #run one chunk with valgrind: |
| 1490 | if [[ -n ${runValgrind} ]]; then |
| 1491 | [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2 |
| 1492 | [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" |
| 1493 | submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}" |
| 1494 | fi |
| fc7e1e45 |
1495 | |
| 5dfac970 |
1496 | ################################################################################ |
| 1497 | ################################################################################ |
| 1498 | # run the CPass0 if requested |
| fc7e1e45 |
1499 | |
| 71353a5f |
1500 | if [ ${runCPass0reco} -eq 1 ]; then |
| fc7e1e45 |
1501 | |
| fc7e1e45 |
1502 | echo |
| 71353a5f |
1503 | echo "starting CPass0... for run ${runNumber}" |
| fc7e1e45 |
1504 | echo |
| fc7e1e45 |
1505 | |
| 5dfac970 |
1506 | # create directory and copy all files that are needed |
| cb473870 |
1507 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" |
| 71353a5f |
1508 | mkdir -p ${targetDirectory} |
| 1509 | |
| cb473870 |
1510 | filesCPass0=( |
| 71353a5f |
1511 | "${configPath}/runCPass0.sh" |
| 1512 | "${configPath}/recCPass0.C" |
| 1513 | "${configPath}/runCalibTrain.C" |
| 1514 | "${configPath}/localOCDBaccessConfig.C" |
| 1515 | "${configPath}/OCDB*.root" |
| 1516 | "${configPath}/sim.C" |
| 1517 | "${configPath}/rec.C" |
| 1518 | "${configPath}/Config.C" |
| cb473870 |
1519 | ) |
| 1520 | for file in ${filesCPass0[*]}; do |
| 1521 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1522 | done |
| fc7e1e45 |
1523 | |
| 71353a5f |
1524 | localInputList=${targetDirectory}/${inputList##*/} |
| 1525 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 5dfac970 |
1526 | # limit nFiles to nMaxChunks |
| 71353a5f |
1527 | nFiles=$(wc -l < ${localInputList}) |
| 1528 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 |
| 1529 | echo "raw files in list: ${nFiles}" |
| 1530 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1531 | nFiles=${nMaxChunks} |
| 5dfac970 |
1532 | fi |
| 71353a5f |
1533 | echo "raw files to process: ${nFiles}" |
| 1534 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1535 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1536 | nFilesToWaitFor=${nFiles} |
| 5dfac970 |
1537 | else |
| 71353a5f |
1538 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| 5dfac970 |
1539 | fi |
| 71353a5f |
1540 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1541 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
1542 | |
| 71353a5f |
1543 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}" |
| fc7e1e45 |
1544 | |
| 5dfac970 |
1545 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 71353a5f |
1546 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'" |
| 1547 | LASTJOB=${JOBID1wait} |
| fc7e1e45 |
1548 | |
| 5dfac970 |
1549 | fi #end running CPass0 |
| 1550 | ################################################################################ |
| fc7e1e45 |
1551 | |
| fc7e1e45 |
1552 | |
| 5dfac970 |
1553 | ################################################################################ |
| 1554 | # submit merging of CPass0, depends on the reconstruction |
| fc7e1e45 |
1555 | |
| 71353a5f |
1556 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
1557 | |
| 5dfac970 |
1558 | echo |
| 71353a5f |
1559 | echo "submit CPass0 merging for run ${runNumber}" |
| 5dfac970 |
1560 | echo |
| fc7e1e45 |
1561 | |
| cb473870 |
1562 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" |
| 71353a5f |
1563 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
1564 | |
| cb473870 |
1565 | #copy the scripts |
| 1566 | filesMergeCPass0=( |
| 71353a5f |
1567 | "${configPath}/OCDB.root" |
| 1568 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 1569 | "${configPath}/mergeMakeOCDB.sh" |
| 1570 | "${configPath}/localOCDBaccessConfig.C" |
| 1571 | "${configPath}/mergeByComponent.C" |
| 1572 | "${configPath}/makeOCDB.C" |
| 1573 | "${configPath}/merge.C" |
| cb473870 |
1574 | ) |
| 1575 | for file in ${filesMergeCPass0[*]}; do |
| 1576 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1577 | done |
| 1578 | |
| 71353a5f |
1579 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}" |
| 1580 | LASTJOB=${JOBID2} |
| fc7e1e45 |
1581 | |
| 71353a5f |
1582 | if [[ -n ${generateMC} ]]; then |
| 1583 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done" |
| 1584 | LASTJOB="mrl${JOBpostfix}" |
| 1585 | fi |
| fc7e1e45 |
1586 | |
| 5dfac970 |
1587 | echo |
| fc7e1e45 |
1588 | fi |
| 5dfac970 |
1589 | # end of merging CPass0 |
| 1590 | ################################################################################ |
| fc7e1e45 |
1591 | |
| 5dfac970 |
1592 | ################################################################################ |
| 1593 | ################################################################################ |
| 1594 | # run the CPass1 if requested |
| fc7e1e45 |
1595 | |
| 71353a5f |
1596 | if [ ${runCPass1reco} -eq 1 ]; then |
| fc7e1e45 |
1597 | |
| cb473870 |
1598 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" |
| 71353a5f |
1599 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done |
| fc7e1e45 |
1600 | |
| 5dfac970 |
1601 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
| 71353a5f |
1602 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" |
| fc7e1e45 |
1603 | |
| 5dfac970 |
1604 | echo |
| 71353a5f |
1605 | echo "starting CPass1... for run ${runNumber}" |
| 5dfac970 |
1606 | echo |
| 1607 | |
| 1608 | # create directory and copy all files that are needed |
| 71353a5f |
1609 | mkdir -p ${targetDirectory} |
| 5dfac970 |
1610 | |
| cb473870 |
1611 | filesCPass1=( |
| 71353a5f |
1612 | "${configPath}/runCPass1.sh" |
| 1613 | "${configPath}/recCPass1.C" |
| 1614 | "${configPath}/recCPass1_OuterDet.C" |
| 1615 | "${configPath}/runCalibTrain.C" |
| 1616 | "${configPath}/QAtrain_duo.C" |
| 1617 | "${configPath}/localOCDBaccessConfig.C" |
| 1618 | "${configPath}/OCDB.root" |
| cb473870 |
1619 | ) |
| 1620 | for file in ${filesCPass1[*]}; do |
| 1621 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1622 | done |
| 5dfac970 |
1623 | |
| 71353a5f |
1624 | if [[ -n ${generateMC} ]]; then |
| 1625 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list |
| 1626 | else |
| 1627 | localInputList=${targetDirectory}/${inputList##*/} |
| 1628 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 1629 | fi |
| 5dfac970 |
1630 | # limit nFiles to nMaxChunks |
| 71353a5f |
1631 | nFiles=$(wc -l < ${localInputList}) |
| 1632 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue |
| 1633 | echo "raw files in list: ${nFiles}" |
| 1634 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1635 | nFiles=${nMaxChunks} |
| 5dfac970 |
1636 | fi |
| 71353a5f |
1637 | echo "raw files to process: ${nFiles}" |
| 1638 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1639 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1640 | nFilesToWaitFor=${nFiles} |
| fc7e1e45 |
1641 | else |
| 71353a5f |
1642 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| fc7e1e45 |
1643 | fi |
| 71353a5f |
1644 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1645 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
1646 | |
| 71353a5f |
1647 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}" |
| fc7e1e45 |
1648 | |
| 5dfac970 |
1649 | ################################################################################ |
| 1650 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 71353a5f |
1651 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'" |
| 1652 | LASTJOB=${JOBID4wait} |
| 5dfac970 |
1653 | ################################################################################ |
| fc7e1e45 |
1654 | |
| 5dfac970 |
1655 | echo |
| 1656 | fi #end running CPass1 |
| fc7e1e45 |
1657 | |
| 5dfac970 |
1658 | ################################################################################ |
| 1659 | # submit merging of CPass1, depends on the reconstruction |
| 71353a5f |
1660 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
1661 | |
| 5dfac970 |
1662 | echo |
| 71353a5f |
1663 | echo "submit CPass1 merging for run ${runNumber}" |
| 5dfac970 |
1664 | echo |
| fc7e1e45 |
1665 | |
| cb473870 |
1666 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" |
| 71353a5f |
1667 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done |
| 1668 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
1669 | |
| cb473870 |
1670 | # copy files |
| 1671 | filesMergeCPass1=( |
| 71353a5f |
1672 | "${configPath}/OCDB.root" |
| 1673 | "${configPath}/localOCDBaccessConfig.C" |
| 1674 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 1675 | "${configPath}/mergeByComponent.C" |
| 1676 | "${configPath}/makeOCDB.C" |
| 1677 | "${configPath}/merge.C" |
| 1678 | "${configPath}/mergeMakeOCDB.sh" |
| 1679 | "${configPath}/QAtrain_duo.C" |
| cb473870 |
1680 | ) |
| 1681 | for file in ${filesMergeCPass1[*]}; do |
| 1682 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1683 | done |
| 1684 | |
| 71353a5f |
1685 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}" |
| 1686 | LASTJOB=${JOBID5} |
| 5dfac970 |
1687 | echo |
| 1688 | fi |
| 1689 | |
| cb473870 |
1690 | ############################## |
| 71353a5f |
1691 | if [ ${runESDfiltering} -eq 1 ]; then |
| 1692 | rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list |
| 1693 | rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done |
| 1694 | echo |
| 1695 | echo submitting filtering for run ${runNumber} |
| 1696 | echo |
| 1697 | submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done " |
| 1698 | submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}" |
| 1699 | LASTJOB=${JOBfilterESDcpass1} |
| cb473870 |
1700 | fi |
| 1701 | |
| 5dfac970 |
1702 | done |
| 1703 | |
| 71353a5f |
1704 | ################################################################################# |
| 1705 | ################################################################################# |
| 1706 | if [ ${runESDfiltering} -eq 1 ]; then |
| 1707 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}" |
| 1708 | else |
| 1709 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}" |
| cb473870 |
1710 | fi |
| 71353a5f |
1711 | LASTJOB=${JOBID5wait} |
| cb473870 |
1712 | |
| 5de3064b |
1713 | ################################################################################# |
| 1714 | echo |
| 1715 | echo "submit make a summary" |
| 1716 | echo |
| 5dfac970 |
1717 | |
| 5de3064b |
1718 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}" |
| 1719 | LASTJOB=${JOBID6} |
| 71353a5f |
1720 | ################################################################################# |
| 5dfac970 |
1721 | |
| 5dfac970 |
1722 | #restore stdout |
| 1723 | exec 1>&7 7>&- |
| 1724 | echo "jobs submitted." |
| 5de3064b |
1725 | return 0 |
| fc7e1e45 |
1726 | } |
| 1727 | |
| cb473870 |
1728 | goWaitForOutput() |
| 1729 | { |
| 1730 | umask 0002 |
| 71353a5f |
1731 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 |
| 1732 | echo searchPath=${1} |
| 1733 | echo fileName=${2} |
| 1734 | echo numberOfFiles=${3} |
| 1735 | echo maxSecondsToWait=${4} |
| 1736 | searchPath=${1} |
| 1737 | fileName=${2} |
| 1738 | numberOfFiles=${3} |
| 1739 | maxSecondsToWait=${4} |
| 1740 | extraFindOptions=${5} |
| 1741 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}" |
| 1742 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) |
| cb473870 |
1743 | while true; do |
| 71353a5f |
1744 | #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l) |
| cb473870 |
1745 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) |
| 71353a5f |
1746 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" |
| 1747 | [[ ${n} -ge ${numberOfFiles} ]] && break |
| 1748 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break |
| cb473870 |
1749 | sleep 60 |
| 1750 | done |
| 1751 | echo "DONE! exiting..." |
| 5de3064b |
1752 | return 0 |
| cb473870 |
1753 | } |
| 1754 | |
| 71353a5f |
1755 | mergeSysLogs() |
| 1756 | { |
| 1757 | outputFile=${1} |
| 1758 | shift |
| 1759 | inputFiles="$@" |
| 1760 | i=0 |
| 1761 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi |
| 1762 | while read x; do |
| 1763 | runNumber=$(guessRunNumber ${x}) |
| 1764 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue |
| 1765 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} |
| 1766 | (( i++ )) |
| 1767 | done < <(ls -1 ${inputFiles}) > ${outputFile} |
| 5de3064b |
1768 | return 0 |
| 71353a5f |
1769 | } |
| 1770 | |
| 1771 | goMakeMergedSummaryTree() |
| 1772 | { |
| 1773 | # create list of calibration entries |
| 1774 | # takes no arguments, just run it in the base output |
| 1775 | # directory with the following files in the working directory |
| 1776 | # |
| 1777 | # Calibration file lists: |
| 1778 | # cpass0.dcsTree.list, cpass1.dcsTree.list |
| 1779 | # QA trending root file: |
| 1780 | # trending.root |
| 1781 | # |
| 1782 | # Production infoo ascii files: |
| 1783 | # summary_pass0.tree |
| 1784 | # summary_pass1.tree |
| 1785 | # |
| 1786 | |
| 1787 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 |
| 1788 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 |
| 1789 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 |
| 1790 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 |
| 1791 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 |
| 1792 | |
| 1793 | #first, dump the C macro to file |
| 1794 | cat << EOF > mergeTree.C |
| 1795 | // |
| 1796 | // Merge summary information |
| 1797 | // Following files are expected to be in the working directory |
| 1798 | // |
| 1799 | // Calibration file lists: |
| 1800 | // cpass0.dcsTree.list, cpass1.dcsTree.list |
| 1801 | // QA trending root file: |
| 1802 | // trending.root |
| 1803 | // |
| 1804 | // Production infoo ascii files: |
| 1805 | // summary_pass0.tree |
| 1806 | // summary_pass1.tree |
| 1807 | // |
| 1808 | void mergeTree(){ |
| 1809 | // |
| 1810 | // |
| 1811 | // |
| 1812 | // Calibration values dump |
| 1813 | // |
| 1814 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); |
| 1815 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); |
| 1816 | // |
| 1817 | // Calibration status dump |
| 1818 | // |
| 1819 | TFile *fprod = TFile::Open("fproduction.root","recreate"); |
| 1820 | TTree tree0, tree1; |
| 1821 | tree0.ReadFile("summary_pass0.tree"); |
| 1822 | tree1.ReadFile("summary_pass1.tree"); |
| 1823 | tree0.Write("CPass0"); |
| 1824 | tree1.Write("CPass1"); |
| 1825 | fprod->Close(); |
| 1826 | // |
| 1827 | // |
| 1828 | // |
| 1829 | TString stringSetup=""; |
| 1830 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // |
| 1831 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // |
| 1832 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // |
| 1833 | // |
| 1834 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // |
| 1835 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // |
| 1836 | // |
| 1837 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); |
| 1838 | } |
| 1839 | EOF |
| 1840 | |
| 1841 | aliroot -b -q "mergeTree.C" > mergeTrees.log |
| 1842 | return $? |
| 1843 | } |
| 1844 | |
| 1845 | goMakeSummary() |
| 1846 | { |
| 1847 | #all the final stuff goes in here for ease of use: |
| 1848 | # summary logs |
| 1849 | # qa plot making |
| 1850 | # final file lists |
| 1851 | |
| 1852 | configFile=${1} |
| 1853 | shift 1 |
| 1854 | extraOpts="$@" |
| 1855 | log="summary.log" |
| 1856 | productionID="qa" |
| 1857 | parseConfig ${configFile} ${extraOpts} |
| 1858 | |
| 1859 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 1860 | |
| 1861 | exec &> >(tee ${log}) |
| 1862 | |
| 1863 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return |
| 1864 | |
| 1865 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 1866 | |
| 1867 | #summarize the global stuff |
| 1868 | echo "env script: ${alirootSource} ${alirootEnv}" |
| 1869 | echo "\$ALICE_ROOT=${ALICE_ROOT}" |
| 1870 | echo "commonOutputPath=${commonOutputPath}" |
| 1871 | |
| 1872 | #summarize the stacktraces |
| 1873 | awk ' |
| 1874 | BEGIN { |
| 1875 | print "frame/I:method/C:line/C:cpass/I:aliroot/I"; |
| 1876 | RS="#[0-9]*"; |
| 1877 | aliroot=0; |
| 1878 | } |
| 1879 | { |
| 1880 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
| 1881 | gsub("#","",RT); |
| 1882 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot |
| 1883 | } |
| 1884 | ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree |
| 1885 | awk ' |
| 1886 | BEGIN { |
| 1887 | RS="#[0-9]*"; |
| 1888 | aliroot=0; |
| 1889 | } |
| 1890 | { |
| 1891 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
| 1892 | gsub("#","",RT); |
| 1893 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot |
| 1894 | } |
| 1895 | ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree |
| 1896 | |
| 1897 | echo total numbers for the production: |
| 1898 | echo |
| 1899 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 1900 | /^calibfile/ {nFiles++;} |
| 1901 | /core dumped/ {nCore++i;} |
| 1902 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null |
| 1903 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1904 | /\/rec.log OK/ {nOK++;} |
| 1905 | /\/rec.log BAD/ {nBAD++;} |
| 1906 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| 1907 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null |
| 1908 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1909 | /\/calib.log OK/ {nOK++;} |
| 1910 | /\/calib.log BAD/ {nBAD++;} |
| 1911 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null |
| 1912 | |
| 1913 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1914 | /merge.log OK/ {nOK++;} |
| 1915 | /merge.log BAD/ {nBAD++;} |
| 1916 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null |
| 1917 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1918 | /ocdb.log OK/ {nOK++;} |
| 1919 | /ocdb.log BAD/ {nBAD++;} |
| 1920 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null |
| 1921 | |
| 1922 | echo |
| 1923 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 1924 | /^calibfile/ {nFiles++;} |
| 1925 | /core dumped/ {nCore++;} |
| 1926 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null |
| 1927 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1928 | /\/rec.log OK/ {nOK++;} |
| 1929 | /\/rec.log BAD/ {nBAD++;} |
| 1930 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| 1931 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null |
| 1932 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1933 | /\/calib.log OK/ {nOK++;} |
| 1934 | /\/calib.log BAD/ {nBAD++;} |
| 1935 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null |
| 1936 | |
| 1937 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1938 | /merge.log OK/ {nOK++;} |
| 1939 | /merge.log BAD/ {nBAD++;} |
| 1940 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null |
| 1941 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 1942 | /ocdb.log OK/ {nOK++;} |
| 1943 | /ocdb.log BAD/ {nBAD++;} |
| 1944 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null |
| 1945 | |
| 1946 | echo |
| 1947 | echo per run stats: |
| 1948 | /bin/ls -1 meta/merge.cpass0.run*.done | while read x |
| 1949 | do |
| 1950 | dir=$(goPrintValues calibfile - ${x}) |
| 1951 | runNumber=$(guessRunNumber ${dir}) |
| 1952 | [[ -z ${runNumber} ]] && continue |
| 1953 | |
| 1954 | if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then |
| 1955 | statusCPass0=( $( |
| 1956 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} |
| 1957 | /\/rec.log OK/ {nOKrec++;} |
| 1958 | /\/rec.log BAD/ {nBADrec++;} |
| 1959 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} |
| 1960 | /stderr OK/ {nOKstderr++;} |
| 1961 | /\/calib.log OK/ {nOKcalib++;} |
| 1962 | /\/calib.log BAD/ {nBADcalib++} |
| 1963 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null |
| 1964 | ) ) |
| 1965 | fi |
| 1966 | |
| 1967 | if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then |
| 1968 | statusCPass1=( $( |
| 1969 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} |
| 1970 | /\/rec.log OK/ {nOKrec++;} |
| 1971 | /\/rec.log BAD/ {nBADrec++;} |
| 1972 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} |
| 1973 | /stderr OK/ {nOKstderr++;} |
| 1974 | /\/calib.log OK/ {nOKcalib++;} |
| 1975 | /\/calib.log BAD/ {nBADcalib++} |
| 1976 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} |
| 1977 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} |
| 1978 | /\/qa_outer.log OK/ {nQAouterOK++;} |
| 1979 | /\/qa_outer.log BAD/ {nQAouterBAD++;} |
| 1980 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null |
| 1981 | ) ) |
| 1982 | fi |
| 1983 | |
| 1984 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) |
| 1985 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 1986 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 1987 | |
| 1988 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} |
| 1989 | done |
| 1990 | |
| 1991 | #make lists with output files - QA, trending, filtering and calibration |
| 1992 | ### wait for the merging of all runs to be over ### |
| 1993 | rm -f qa.list |
| 1994 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1995 | rm -f calib.list |
| 1996 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1997 | rm -f trending.list |
| 1998 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 1999 | rm -f filtering.list |
| 2000 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null |
| 2001 | #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list |
| 2002 | rm -f cpass0.dcsTree.list |
| 2003 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null |
| 2004 | #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list |
| 2005 | rm -f cpass1.dcsTree.list |
| 2006 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 2007 | |
| 2008 | #merge trending |
| 2009 | rm -f ${commonOutputPath}/trending_merged.root |
| 2010 | goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log |
| 2011 | |
| 2012 | goMakeSummaryTree ${commonOutputPath} 0 |
| 2013 | goMakeSummaryTree ${commonOutputPath} 1 |
| 2014 | |
| 2015 | goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log |
| 2016 | |
| 2017 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees |
| 2018 | goMakeMergedSummaryTree |
| 2019 | |
| 2020 | #if set, email the summary |
| 5de3064b |
2021 | [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO} |
| 71353a5f |
2022 | |
| 2023 | return 0 |
| 2024 | } |
| 2025 | |
| 2026 | goMakeSummaryTree() |
| 2027 | { |
| 2028 | if [[ $# -lt 1 ]] ; then |
| 2029 | return |
| 2030 | fi |
| 2031 | #1. define vars/arrays |
| 2032 | DIR=${1} #use input or exec in current dir |
| 2033 | pass=${2-"0"} #pass from input |
| 2034 | outfile="summary_pass${pass}.tree" |
| 2035 | Ncolumns=0 |
| 2036 | test -f ${outfile} && : >${outfile} |
| 2037 | errfile=${outfile/tree/err} |
| 2038 | test -f ${errfile} && : >${errfile} |
| 2039 | |
| 2040 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) |
| 2041 | Ncounter=${#counterString[@]} |
| 2042 | |
| 2043 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) |
| 2044 | Nstatus=${#statusString[@]} |
| 2045 | |
| 2046 | |
| 2047 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) |
| 2048 | Nrates=${#ratesString[@]} |
| 2049 | |
| 2050 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) |
| 2051 | Nruns=${#runs[@]} |
| 2052 | |
| 2053 | echo -n runnumber/I >>${outfile} |
| 2054 | echo -n :cpass${pass}status/I >>${outfile} |
| 2055 | echo -n :cpass${pass}QAstatus/I >>${outfile} |
| 2056 | for i in ${ratesString[@]}; do |
| 2057 | echo -n :${i}OK/I >>${outfile} |
| 2058 | echo -n :${i}BAD/I >>${outfile} |
| 2059 | |
| 2060 | done |
| 2061 | for i in ${counterString[@]} ${statusString[@]} ; do |
| 2062 | echo -n :${i}/I >>${outfile} |
| 2063 | done |
| 2064 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) |
| 2065 | echo >> ${outfile} |
| 2066 | |
| 2067 | #2. loop runs |
| 2068 | |
| 2069 | for runnumber in ${runs[@]} ; do |
| 2070 | |
| 2071 | |
| 2072 | |
| 2073 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" |
| 2074 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" |
| 2075 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') |
| 2076 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then |
| 2077 | echo "${filemerge} does not exist!" >>${errfile} |
| 2078 | continue |
| 2079 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then |
| 2080 | echo "${filejob} does not exist!" >>${errfile} |
| 2081 | echo -n ${runnumber} >> ${outfile} |
| 2082 | for i in $(seq ${Ncolumns}) ; do |
| 2083 | echo -n "-1" >> ${outfile} |
| 2084 | done |
| 2085 | echo >> ${outfile} |
| 2086 | continue |
| 2087 | fi |
| 2088 | echo -n ${runnumber} >> ${outfile} |
| 2089 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' |
| 2090 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) |
| 2091 | echo -n " ${passStatus}" >> ${outfile} |
| 2092 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) |
| 2093 | echo -n " ${qaStatus}" >> ${outfile} |
| 2094 | |
| 2095 | |
| 2096 | #fill OK/BAD rates |
| 2097 | for i in $(seq 0 $((${Nrates}-1))) ; do |
| 2098 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) |
| 2099 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) |
| 2100 | |
| 2101 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then |
| 2102 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) |
| 2103 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) |
| 2104 | fi |
| 2105 | echo -n " ${var1}" >> ${outfile} |
| 2106 | echo -n " ${var2}" >> ${outfile} |
| 2107 | done |
| 2108 | |
| 2109 | if [[ -f ${fileOCDB} ]] ; then |
| 2110 | #fill counter |
| 2111 | for i in $(seq 0 $((${Ncounter}-1))) ; do |
| 2112 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) |
| 2113 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2114 | done |
| 2115 | |
| 2116 | #fill status |
| 2117 | for i in $(seq 0 $((${Nstatus}-1))) ; do |
| 2118 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) |
| 2119 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2120 | done |
| 2121 | fi |
| 2122 | echo >> ${outfile} |
| 2123 | done |
| 2124 | |
| 5de3064b |
2125 | return 0 |
| 71353a5f |
2126 | } |
| 2127 | |
| 2128 | parseConfig() |
| 2129 | { |
| 2130 | configFile=${1} |
| 2131 | shift |
| 2132 | |
| 2133 | #first, source the config file |
| 2134 | if [ -f ${configFile} ]; then |
| 2135 | source ${configFile} |
| 2136 | else |
| 2137 | echo "config file ${configFile} not found!, skipping..." |
| 2138 | fi |
| 2139 | |
| 2140 | #then, parse the options as theya override the optionf from file |
| 2141 | while [[ -n ${1} ]]; do |
| 2142 | var="${1#--}" |
| 2143 | echo "extraOpts: exporting ${var}" |
| 2144 | eval "export ${var}" |
| 2145 | shift |
| 2146 | done |
| 2147 | |
| 2148 | #export the aliroot function if defined to override normal behaviour |
| 2149 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot |
| 5de3064b |
2150 | return 0 |
| 71353a5f |
2151 | } |
| 2152 | |
| 2153 | aliroot() |
| 2154 | { |
| 2155 | args="$@" |
| 2156 | if [[ -n ${valgrindCommand} ]]; then |
| 2157 | valgrindLogFile="cpu.txt" |
| 2158 | [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt" |
| 2159 | [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt" |
| 2160 | [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}" |
| 2161 | echo running ${valgrindCommand} aliroot ${args} |
| 2162 | ${valgrindCommand} aliroot ${args} |
| 2163 | else |
| 2164 | #to prevent an infinite recursion use "command aliroot" to disable |
| 2165 | #aliases and functions |
| 2166 | echo running command aliroot ${args} |
| 2167 | command aliroot ${args} |
| 2168 | fi |
| 5de3064b |
2169 | return 0 |
| 71353a5f |
2170 | } |
| 2171 | |
| 2172 | guessRunData() |
| 2173 | { |
| 2174 | #guess the period from the path, pick the rightmost one |
| 2175 | period="" |
| 2176 | runNumber="" |
| 2177 | year="" |
| 2178 | pass="" |
| 2179 | legoTrainRunNumber="" |
| 2180 | dataType="" |
| 2181 | |
| 2182 | local shortRunNumber="" |
| 2183 | local IFS="/" |
| 2184 | declare -a path=( $1 ) |
| 2185 | local dirDepth=$(( ${#path[*]}-1 )) |
| 2186 | i=0 |
| 2187 | #for ((x=${dirDepth};x>=0;x--)); do |
| 2188 | for ((x=0;x<=${dirDepth};x++)); do |
| 2189 | |
| 2190 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} |
| 2191 | local field=${path[${x}]} |
| 2192 | local fieldNext=${path[$((x+1))]} |
| 2193 | |
| 2194 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} |
| 2195 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} |
| 2196 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} |
| 2197 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} |
| 2198 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} |
| 2199 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} |
| 2200 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} |
| 2201 | (( i++ )) |
| 2202 | done |
| 2203 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} |
| 2204 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} |
| 2205 | |
| 2206 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; |
| 2207 | if [[ -z ${runNumber}} ]]; |
| 2208 | then |
| 2209 | #error condition |
| 2210 | return 1 |
| 2211 | else |
| 2212 | #ALL OK |
| 2213 | return 0 |
| 2214 | fi |
| 5de3064b |
2215 | return 0 |
| 71353a5f |
2216 | } |
| 2217 | |
| fc7e1e45 |
2218 | main "$@" |
git://git.uio.no / u / mrichter / AliRoot.git / blame