| fc7e1e45 |
1 | #!/bin/bash |
| 71353a5f |
2 | #include benchmark.config |
| 3 | |
| b802abe4 |
4 | # blame: Mikolaj Krzewicki, mkrzewic@cern.ch |
| fc7e1e45 |
5 | # this script runs the CPass0/CPass1 train |
| 6 | # produced OCDB updates are local |
| 7 | |
| 8 | main() |
| 9 | { |
| 10 | #run in proper mode depending on the selection |
| 71353a5f |
11 | if [[ $# -lt 1 ]]; then |
| 12 | if [[ ! "${0}" =~ "bash" ]]; then |
| 13 | echo "uses makeflow:" |
| 14 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" |
| 15 | echo "uses a batch system (SGE):" |
| 16 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" |
| 17 | echo "extraOpts if non-empty override the config file, e.g.:" |
| 9510565c |
18 | echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10" |
| 71353a5f |
19 | fi |
| 20 | return |
| 21 | fi |
| 22 | |
| 23 | #define some aliases - default is to call one of the functions directly |
| 24 | runMode=${1} |
| fc7e1e45 |
25 | umask 0002 |
| 26 | shift |
| 71353a5f |
27 | case ${runMode} in |
| fc7e1e45 |
28 | "CPass0") goCPass0 "$@";; |
| 29 | "CPass1") goCPass1 "$@";; |
| 71353a5f |
30 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; |
| fc7e1e45 |
31 | "MergeCPass0") goMergeCPass0 "$@";; |
| 32 | "MergeCPass1") goMergeCPass1 "$@";; |
| cb473870 |
33 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; |
| 71353a5f |
34 | "MakeSummary") goMakeSummary "$@";; |
| cb473870 |
35 | "run") goSubmitMakeflow "$@";; |
| 36 | "submit") goSubmitBatch "$@";; |
| fc7e1e45 |
37 | "test") goTest "$@";; |
| 71353a5f |
38 | "GenerateMakeflow") goGenerateMakeflow "$@";; |
| 39 | "PrintValues") goPrintValues "$@";; |
| cb473870 |
40 | "CreateQAplots") goCreateQAplots "$@";; |
| 41 | "WaitForOutput") goWaitForOutput "$@";; |
| 71353a5f |
42 | "Merge") goMerge "$@";; |
| 43 | *) |
| 444b9916 |
44 | ${runMode} "$@" |
| fc7e1e45 |
45 | ;; |
| 46 | esac |
| 343130c8 |
47 | return $? |
| fc7e1e45 |
48 | } |
| 49 | |
| 71353a5f |
50 | generateMC() |
| 51 | { |
| 52 | #generate one raw chunk in current directory |
| 53 | SEED=${JOB_ID}${SGE_TASK_ID} |
| 54 | export CONFIG_SEED=${SEED} |
| 55 | runNumber=${1} |
| 56 | OCDBpath=${2} |
| 57 | nEventsim=${3} |
| 58 | if [[ -n ${pretend} ]]; then |
| 0eaf38c7 |
59 | sleep ${pretendDelay} |
| 71353a5f |
60 | touch galice.root |
| 61 | else |
| 62 | if [[ -f sim.C && -f Config.C ]] ; then |
| 63 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 |
| 64 | mv syswatch.log simwatch.log |
| 65 | fi |
| 66 | fi |
| 67 | } |
| 68 | |
| fc7e1e45 |
69 | goCPass0() |
| 78d0bbfd |
70 | ( |
| fc7e1e45 |
71 | umask 0002 |
| 72 | |
| 71353a5f |
73 | targetDirectory=${1} |
| 74 | inputList=${2} |
| 75 | nEvents=${3} |
| 76 | ocdbPath=${4} |
| 77 | configFile=${5} |
| 78 | runNumber=${6} |
| 79 | jobindex=${7} |
| 80 | shift 7 |
| 6fc48f9e |
81 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 71353a5f |
82 | |
| 39527a15 |
83 | #record the working directory provided by the batch system |
| 84 | batchWorkingDirectory=${PWD} |
| 85 | |
| 71353a5f |
86 | #use the jobindex only if set and non-negative |
| 87 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 88 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 89 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 999a843d |
90 | if [[ -z ${jobindex} ]]; then |
| 91 | echo "no jobindex!" |
| 92 | return 1 |
| 93 | fi |
| 71353a5f |
94 | fi |
| 95 | |
| 96 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| ef77d296 |
97 | |
| 98 | # This file signals that/if everything went fine |
| 99 | doneFileBase="cpass0.job${jobindex}.run${runNumber}.done" |
| 100 | [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done" |
| 101 | |
| 102 | # We will have two copies of the file |
| 103 | mkdir -p "${commonOutputPath}/meta" || return 1 |
| 104 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" |
| 105 | doneFile="${commonOutputPath}/meta/${doneFileBase}" |
| 71353a5f |
106 | |
| 107 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 108 | |
| 109 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 110 | ulimit -c unlimited |
| 111 | export ALIROOT_FORCE_COREDUMP |
| 112 | fi |
| fc7e1e45 |
113 | |
| cb473870 |
114 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 115 | #at the QA stage the pass number is guessed from the path stored here. |
| 116 | #The Format is: |
| 117 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 118 | export PRODUCTION_METADATA="OutputDir=cpass0" |
| 119 | |
| 5de3064b |
120 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 121 | infile=${inputList} |
| 122 | else |
| 123 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| 124 | fi |
| 71353a5f |
125 | chunkName=${infile##*/} |
| 71353a5f |
126 | |
| 127 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 128 | mkdir -p ${outputDir} |
| 1795556a |
129 | if [[ ! -d ${outputDir} ]]; then |
| ef77d296 |
130 | touch ${doneFileTmp} |
| 131 | echo "cannot make ${outputDir}" >> ${doneFileTmp} |
| 132 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 133 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
134 | return 1 |
| 135 | fi |
| 71353a5f |
136 | |
| 71353a5f |
137 | runpath=${outputDir} |
| 0eaf38c7 |
138 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX) |
| e91a9c78 |
139 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} |
| 71353a5f |
140 | mkdir -p ${runpath} |
| 1795556a |
141 | if [[ ! -d ${runpath} ]]; then |
| ef77d296 |
142 | touch ${doneFileTmp} |
| 143 | echo "cannot make runpath ${runpath}" >> ${doneFileTmp} |
| 144 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 145 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
146 | return 1 |
| 147 | fi |
| 0eaf38c7 |
148 | if ! cd ${runpath}; then |
| ef77d296 |
149 | touch ${doneFileTmp} |
| 150 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} |
| 151 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 152 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 0eaf38c7 |
153 | return 1 |
| 154 | fi |
| fc7e1e45 |
155 | |
| 5de3064b |
156 | #runCPassX/C expects the raw chunk to be linked in the run dir |
| 157 | #despite it being accessed by the full path |
| 158 | ln -s ${infile} ${runpath}/${chunkName} |
| fc7e1e45 |
159 | |
| 71353a5f |
160 | #####MC |
| 161 | if [[ -n ${generateMC} ]]; then |
| 162 | olddir=${PWD} |
| 163 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} |
| 164 | simrunpath=${outputDirMC} |
| 0eaf38c7 |
165 | #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} |
| 166 | [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX) |
| 71353a5f |
167 | mkdir -p ${outputDirMC} |
| 168 | mkdir -p ${simrunpath} |
| 0eaf38c7 |
169 | if cd ${simrunpath}; then |
| fc7e1e45 |
170 | |
| 0eaf38c7 |
171 | filesMC=( |
| 172 | "${batchWorkingDirectory}/sim.C" |
| 173 | "${batchWorkingDirectory}/rec.C" |
| 174 | "${batchWorkingDirectory}/Config.C" |
| 175 | "${batchWorkingDirectory}/OCDB_*.root" |
| 176 | ) |
| 177 | for file in ${filesMC[*]}; do |
| 178 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 179 | done |
| fc7e1e45 |
180 | |
| 0eaf38c7 |
181 | generateMC ${runNumber} ${ocdbPath} ${nEvents} |
| 182 | |
| 183 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works |
| 184 | cd ${olddir} |
| 185 | |
| 186 | ln -s ${outputDirMC}/* ${runpath}/ |
| 187 | |
| 188 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! |
| 189 | infile="" |
| 190 | fi |
| 71353a5f |
191 | fi |
| 192 | ###### |
| 193 | |
| 1795556a |
194 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then |
| ef77d296 |
195 | touch ${doneFileTmp} |
| 196 | echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp} |
| 197 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 198 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
199 | return 1 |
| 200 | fi |
| 71353a5f |
201 | |
| 202 | logOutputDir=${runpath} |
| 203 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 0eaf38c7 |
204 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
| 205 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 71353a5f |
206 | echo "${0} $*" |
| fc7e1e45 |
207 | |
| 208 | echo "#####################" |
| 209 | echo CPass0: |
| 210 | echo JOB setup |
| ef77d296 |
211 | echo nEvents ${nEvents} |
| 212 | echo runNumber ${runNumber} |
| 213 | echo ocdbPath ${ocdbPath} |
| 214 | echo infile ${infile} |
| 215 | echo chunkName ${chunkName} |
| 216 | echo jobindex ${jobindex} |
| 217 | echo recoTriggerOptions ${recoTriggerOptions} |
| 218 | echo targetDirectory ${targetDirectory} |
| 219 | echo commonOutputPath ${commonOutputPath} |
| 220 | echo doneFile ${doneFile} |
| 221 | echo batchWorkingDirectory ${batchWorkingDirectory} |
| 222 | echo runpath ${runpath} |
| 223 | echo outputDir ${outputDir} |
| 224 | echo PWD ${PWD} |
| 225 | echo ALICE_ROOT ${ALICE_ROOT} |
| fc7e1e45 |
226 | echo "########## ###########" |
| 227 | |
| 71353a5f |
228 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
229 | |
| 230 | filesCPass0=( |
| 39527a15 |
231 | "${batchWorkingDirectory}/runCPass0.sh" |
| 232 | "${batchWorkingDirectory}/recCPass0.C" |
| 233 | "${batchWorkingDirectory}/runCalibTrain.C" |
| 234 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" |
| 235 | "${batchWorkingDirectory}/OCDB.root" |
| 71353a5f |
236 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
| 237 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" |
| 238 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" |
| fc7e1e45 |
239 | ) |
| 240 | |
| 241 | for file in ${filesCPass0[*]}; do |
| 242 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 1a0e38e3 |
243 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
| fc7e1e45 |
244 | done |
| 245 | |
| 71353a5f |
246 | echo "this directory (${PWD}) contents:" |
| 247 | /bin/ls |
| fc7e1e45 |
248 | echo |
| 249 | chmod u+x runCPass0.sh |
| 250 | |
| 71353a5f |
251 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 252 | |
| 253 | if [[ -n ${postSetUpActionCPass0} ]]; then |
| 254 | echo "running ${postSetUpActionCPass0}" |
| 255 | eval ${postSetUpActionCPass0} |
| fc7e1e45 |
256 | fi |
| 257 | |
| 258 | #run CPass0 |
| 71353a5f |
259 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 260 | if [[ -n ${pretend} ]]; then |
| 0eaf38c7 |
261 | sleep ${pretendDelay} |
| cb473870 |
262 | touch AliESDs.root |
| 263 | touch AliESDfriends.root |
| fc7e1e45 |
264 | touch AliESDfriends_v1.root |
| 265 | touch rec.log |
| 266 | touch calib.log |
| 267 | else |
| 71353a5f |
268 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| 269 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| fc7e1e45 |
270 | fi |
| 271 | |
| fc7e1e45 |
272 | #move stuff to final destination |
| 71353a5f |
273 | echo "this directory (${PWD}) contents:" |
| 274 | /bin/ls |
| fc7e1e45 |
275 | echo |
| 276 | |
| ef77d296 |
277 | # [dberzano] OK this is fine! |
| 71353a5f |
278 | echo rm -f ./${chunkName} |
| 279 | rm -f ./${chunkName} |
| 13abb339 |
280 | echo "paranoidCp ${runpath}/* ${outputDir}" |
| 281 | paranoidCp ${runpath}/* ${outputDir} |
| fc7e1e45 |
282 | echo |
| 5ed7c82c |
283 | |
| 284 | #validate CPass0 |
| 285 | cd ${outputDir} |
| ef77d296 |
286 | if summarizeLogs >> ${doneFileTmp}; then |
| 287 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp} |
| 288 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp} |
| 289 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp} |
| ab946185 |
290 | fi |
| fc7e1e45 |
291 | |
| 0eaf38c7 |
292 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}" |
| ef77d296 |
293 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 294 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 5de3064b |
295 | return 0 |
| 78d0bbfd |
296 | ) |
| fc7e1e45 |
297 | |
| 298 | goCPass1() |
| 78d0bbfd |
299 | ( |
| fc7e1e45 |
300 | umask 0002 |
| 301 | |
| 71353a5f |
302 | targetDirectory=${1} |
| 303 | inputList=${2} |
| 304 | nEvents=${3} |
| 305 | ocdbPath=${4} |
| 306 | configFile=${5} |
| 307 | runNumber=${6} |
| 308 | jobindex=${7} |
| 309 | shift 7 |
| 444b9916 |
310 | extraOpts=("$@") |
| 6fc48f9e |
311 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 71353a5f |
312 | |
| 39527a15 |
313 | #record the working directory provided by the batch system |
| 314 | batchWorkingDirectory=${PWD} |
| 315 | |
| 71353a5f |
316 | #use the jobindex only if set and non-negative |
| 317 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then |
| 318 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} |
| 319 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} |
| 999a843d |
320 | if [[ -z ${jobindex} ]]; then |
| 321 | echo "no jobindex!" |
| 322 | return 1 |
| 323 | fi |
| 71353a5f |
324 | fi |
| 325 | |
| 326 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| ef77d296 |
327 | |
| 328 | # This file signals that/if everything went fine |
| 329 | doneFileBase="cpass1.job${jobindex}.run${runNumber}.done" |
| 330 | [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done" |
| 331 | |
| 332 | # We will have two copies of the file |
| 333 | mkdir -p "${commonOutputPath}/meta" || return 1 |
| 334 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" |
| 335 | doneFile="${commonOutputPath}/meta/${doneFileBase}" |
| 71353a5f |
336 | |
| 337 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 338 | |
| 339 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then |
| 340 | ulimit -c unlimited |
| 341 | export ALIROOT_FORCE_COREDUMP |
| 342 | fi |
| fc7e1e45 |
343 | |
| cb473870 |
344 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
| 345 | #at the QA stage the pass number is guessed from the path stored here. |
| 346 | #The Format is: |
| 347 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= |
| 348 | export PRODUCTION_METADATA="OutputDir=cpass1" |
| 349 | |
| 1795556a |
350 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then |
| ef77d296 |
351 | touch ${doneFileTmp} |
| 352 | echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp} |
| 353 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 354 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
355 | return 1 |
| 356 | fi |
| fc7e1e45 |
357 | if [[ "${inputList}" =~ \.root$ ]]; then |
| 71353a5f |
358 | infile=${inputList} |
| fc7e1e45 |
359 | else |
| 71353a5f |
360 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
| fc7e1e45 |
361 | fi |
| fc7e1e45 |
362 | chunkName=${infile##*/} |
| 5de3064b |
363 | |
| 71353a5f |
364 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
| 365 | mkdir -p ${outputDir} |
| 1795556a |
366 | if [[ ! -d ${outputDir} ]];then |
| ef77d296 |
367 | touch ${doneFileTmp} |
| 368 | echo "cannot make ${outputDir}" >> ${doneFileTmp} |
| 369 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 370 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
371 | return 1 |
| 372 | fi |
| fc7e1e45 |
373 | |
| 71353a5f |
374 | runpath=${outputDir} |
| 0eaf38c7 |
375 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX) |
| e91a9c78 |
376 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} |
| 71353a5f |
377 | |
| 378 | #MC |
| 379 | if [[ "${infile}" =~ galice\.root ]]; then |
| 380 | ln -s ${inputList%/*}/* ${runpath} |
| 381 | infile="" |
| 382 | fi |
| fc7e1e45 |
383 | |
| 384 | #init the running path |
| 71353a5f |
385 | mkdir -p ${runpath} |
| 1795556a |
386 | if [[ ! -d ${runpath} ]]; then |
| ef77d296 |
387 | touch ${doneFileTmp} |
| 388 | echo "cannot make runpath ${runpath}" >> ${doneFileTmp} |
| 389 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 390 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
391 | return 1 |
| 392 | fi |
| 0eaf38c7 |
393 | if ! cd ${runpath}; then |
| ef77d296 |
394 | touch ${doneFileTmp} |
| 395 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} |
| 396 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 397 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 0eaf38c7 |
398 | return 1 |
| 399 | fi |
| fc7e1e45 |
400 | |
| 71353a5f |
401 | #this is needed for runCPass1.sh |
| 402 | ln -s ${infile} ${runpath}/${chunkName} |
| fc7e1e45 |
403 | |
| 71353a5f |
404 | logOutputDir=${runpath} |
| 405 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 0eaf38c7 |
406 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
| 407 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr |
| 71353a5f |
408 | echo "${0} $*" |
| fc7e1e45 |
409 | |
| 410 | echo "#####################" |
| 411 | echo CPass1: |
| 412 | echo JOB setup |
| ef77d296 |
413 | echo nEvents ${nEvents} |
| 414 | echo runNumber ${runNumber} |
| 415 | echo ocdbPath ${ocdbPath} |
| 416 | echo infile ${infile} |
| 417 | echo chunkName ${chunkName} |
| 418 | echo jobindex ${jobindex} |
| 419 | echo recoTriggerOptions ${recoTriggerOptions} |
| 420 | echo targetDirectory ${targetDirectory} |
| 421 | echo commonOutputPath ${commonOutputPath} |
| 422 | echo doneFile ${doneFile} |
| 423 | echo runpath ${runpath} |
| 424 | echo outputDir ${outputDir} |
| 1795556a |
425 | echo batchWorkingDirectory ${batchWorkingDirectory} |
| ef77d296 |
426 | echo ALICE_ROOT ${ALICE_ROOT} |
| 427 | echo PWD ${PWD} |
| 428 | echo "#####################" |
| fc7e1e45 |
429 | |
| 71353a5f |
430 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
431 | |
| 432 | filesCPass1=( |
| 39527a15 |
433 | "${batchWorkingDirectory}/runCPass1.sh" |
| 434 | "${batchWorkingDirectory}/recCPass1.C" |
| 435 | "${batchWorkingDirectory}/recCPass1_OuterDet.C" |
| 436 | "${batchWorkingDirectory}/runCalibTrain.C" |
| 437 | "${batchWorkingDirectory}/QAtrain_duo.C" |
| 438 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" |
| 0eaf38c7 |
439 | "${batchWorkingDirectory}/${configFile}" |
| 617f8886 |
440 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 39527a15 |
441 | "${batchWorkingDirectory}/OCDB.root" |
| 71353a5f |
442 | "${trustedQAtrainMacro}" |
| 443 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
| 444 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" |
| 445 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" |
| 446 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" |
| 447 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
448 | ) |
| 449 | |
| 78d0bbfd |
450 | for file in "${filesCPass1[@]}"; do |
| fc7e1e45 |
451 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 78d0bbfd |
452 | [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/} |
| fc7e1e45 |
453 | done |
| 454 | |
| 71353a5f |
455 | echo "this directory (${PWD}) contents:" |
| 456 | /bin/ls |
| fc7e1e45 |
457 | echo |
| 458 | |
| 78d0bbfd |
459 | #remove spaces around commas from calls to root |
| 71353a5f |
460 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 461 | |
| 462 | if [[ -n ${postSetUpActionCPass1} ]]; then |
| 463 | echo "running ${postSetUpActionCPass1}" |
| 464 | eval ${postSetUpActionCPass1} |
| fc7e1e45 |
465 | echo |
| 466 | fi |
| 467 | |
| 468 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 469 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
470 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 471 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
472 | else |
| 473 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 474 | fi |
| 475 | |
| 71353a5f |
476 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then |
| ef77d296 |
477 | touch ${doneFileTmp} |
| 478 | echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp} |
| 479 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 480 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 71353a5f |
481 | return 1 |
| cb473870 |
482 | fi |
| fc7e1e45 |
483 | |
| 71353a5f |
484 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory |
| 485 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB |
| 486 | #entries using a relative path, e.g. local://./OCDB |
| 78d0bbfd |
487 | echo "linking the OCDB/ for Barrel and OuterDet directories" |
| 71353a5f |
488 | mkdir Barrel OuterDet |
| 78d0bbfd |
489 | ls -l |
| 71353a5f |
490 | ln -s ../OCDB Barrel/OCDB |
| 491 | ln -s ../OCDB OuterDet/OCDB |
| 492 | |
| 444b9916 |
493 | #setup the filtering |
| 494 | #the following option enables the filtering task inside the QAtrain_duo.C |
| 495 | [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1 |
| 496 | #set the downscaling factors during the filtering fro expert QA (overrides the previous values) |
| 497 | if [[ -n ${filteringFactorHighPt} ]]; then |
| 498 | export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt} |
| 499 | fi |
| 500 | if [[ -n ${filteringFactorV0s} ]]; then |
| 501 | export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} |
| 502 | fi |
| 503 | |
| fc7e1e45 |
504 | #run CPass1 |
| 505 | chmod u+x runCPass1.sh |
| 71353a5f |
506 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
| 507 | if [[ -n ${pretend} ]]; then |
| 0eaf38c7 |
508 | sleep ${pretendDelay} |
| cb473870 |
509 | touch AliESDs_Barrel.root |
| 510 | touch AliESDfriends_Barrel.root |
| fc7e1e45 |
511 | touch AliESDfriends_v1.root |
| cb473870 |
512 | touch QAresults_barrel.root |
| 513 | touch EventStat_temp_barrel.root |
| 514 | touch AODtpITS.root |
| 515 | touch AliESDs_Outer.root |
| 516 | touch AliESDfriends_Outer.root |
| 517 | touch QAresults_outer.root |
| 518 | touch EventStat_temp_outer.root |
| fc7e1e45 |
519 | touch rec.log |
| 520 | touch calib.log |
| 521 | touch qa.log |
| 444b9916 |
522 | touch filtering.log FilterEvents_Trees.root |
| fc7e1e45 |
523 | else |
| 71353a5f |
524 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
| 444b9916 |
525 | |
| 526 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root |
| 527 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root |
| 528 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . |
| 529 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . |
| 530 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root |
| 531 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root |
| 532 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . |
| 533 | [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root . |
| 534 | |
| 535 | #make the filtered tree (if requested and not already produced by QA |
| 536 | [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list |
| fab31805 |
537 | if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then |
| 538 | goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log |
| 539 | else |
| 540 | echo "" |
| 444b9916 |
541 | fi |
| 542 | |
| fc7e1e45 |
543 | fi |
| cb473870 |
544 | |
| 545 | ##handle possible crashes in QA (happens often in trunk) |
| 546 | ##rerun QA with a trusted aliroot version |
| 547 | #if [[ $(validateLog qa_barrel.log) ]]; then |
| 548 | # echo "qa_barrel.log not validated!" |
| 549 | #fi |
| 550 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then |
| 71353a5f |
551 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" |
| cb473870 |
552 | # source ${setupTrustedAliROOTenvInCurrentShell} |
| 553 | # cd Barrel |
| 554 | # rm QAresults_barrel.root |
| 555 | # rm EventStat_temp_barrel.root |
| 556 | # rm AODtpITS.root |
| 557 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root |
| 558 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root |
| 71353a5f |
559 | # if [[ -n ${trustedQAtrainMacro} ]]; then |
| 560 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" |
| cb473870 |
561 | # fi |
| 71353a5f |
562 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" |
| 563 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log |
| cb473870 |
564 | # cd ../ |
| 565 | #fi |
| cb473870 |
566 | |
| fc7e1e45 |
567 | #move stuff to final destination |
| 71353a5f |
568 | echo "this directory (${PWD}) contents:" |
| 569 | /bin/ls |
| 570 | echo rm -f ./${chunkName} |
| 571 | rm -f ./${chunkName} |
| 13abb339 |
572 | echo "paranoidCp ${runpath}/* ${outputDir}" |
| 573 | paranoidCp ${runpath}/* ${outputDir} |
| fc7e1e45 |
574 | echo |
| 575 | |
| 5ed7c82c |
576 | #validate CPass1 |
| 577 | cd ${outputDir} |
| ef77d296 |
578 | if summarizeLogs >> ${doneFileTmp}; then |
| 579 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp} |
| 580 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp} |
| 581 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp} |
| 582 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp} |
| 583 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp} |
| 584 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp} |
| 585 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} |
| 922d0b10 |
586 | else |
| ef77d296 |
587 | if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then |
| 588 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp} |
| 589 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp} |
| 922d0b10 |
590 | fi |
| ef77d296 |
591 | if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then |
| 592 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp} |
| 593 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp} |
| 922d0b10 |
594 | fi |
| ef77d296 |
595 | if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then |
| 596 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} |
| 922d0b10 |
597 | fi |
| ab946185 |
598 | fi |
| 599 | |
| 71353a5f |
600 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| ef77d296 |
601 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 602 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 5de3064b |
603 | return 0 |
| 78d0bbfd |
604 | ) |
| fc7e1e45 |
605 | |
| 606 | |
| 607 | goMergeCPass0() |
| 78d0bbfd |
608 | ( |
| fc7e1e45 |
609 | # |
| 610 | # find the output files and merge them |
| 611 | # |
| 612 | |
| 71353a5f |
613 | outputDir=${1} |
| 614 | ocdbStorage=${2} |
| 615 | configFile=${3} |
| 616 | runNumber=${4} |
| 617 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly |
| 618 | shift 5 |
| 6fc48f9e |
619 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 71353a5f |
620 | |
| 39527a15 |
621 | #record the working directory provided by the batch system |
| 622 | batchWorkingDirectory=${PWD} |
| 623 | |
| 71353a5f |
624 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| ef77d296 |
625 | |
| 626 | # This file signals that everything went fine |
| 627 | doneFileBase="merge.cpass0.run${runNumber}.done" |
| 628 | |
| 629 | # We will have two copies of the file |
| 630 | mkdir -p "${commonOutputPath}/meta" || return 1 |
| 631 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" |
| 632 | doneFile="${commonOutputPath}/meta/${doneFileBase}" |
| 71353a5f |
633 | |
| 634 | umask 0002 |
| 635 | ulimit -c unlimited |
| fc7e1e45 |
636 | |
| 71353a5f |
637 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
638 | |
| 71353a5f |
639 | runpath=${outputDir} |
| 0eaf38c7 |
640 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX) |
| e91a9c78 |
641 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber} |
| fc7e1e45 |
642 | |
| 71353a5f |
643 | mkdir -p ${runpath} |
| 1795556a |
644 | if [[ ! -d ${runpath} ]]; then |
| ef77d296 |
645 | touch ${doneFileTmp} |
| 646 | echo "not able to make the runpath ${runpath}" >> ${doneFileTmp} |
| 647 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 648 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
649 | return 1 |
| 650 | fi |
| 0eaf38c7 |
651 | if ! cd ${runpath}; then |
| ef77d296 |
652 | touch ${doneFileTmp} |
| 653 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} |
| 654 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 655 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 0eaf38c7 |
656 | return 1 |
| 657 | fi |
| fc7e1e45 |
658 | |
| 71353a5f |
659 | logOutputDir=${runpath} |
| 660 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 1795556a |
661 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
| 71353a5f |
662 | echo "${0} $*" |
| fc7e1e45 |
663 | |
| fc7e1e45 |
664 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| fc7e1e45 |
665 | |
| 666 | echo goMergeCPass0 SETUP: |
| 71353a5f |
667 | echo runNumber=${runNumber} |
| 668 | echo outputDir=${outputDir} |
| 669 | echo ocdbStorage=${ocdbStorage} |
| 670 | echo calibrationFilesToMerge=${calibrationFilesToMerge} |
| 671 | echo mergingScript=${mergingScript} |
| 672 | echo commonOutputPath=${commonOutputPath} |
| 673 | echo runpath=${runpath} |
| fc7e1e45 |
674 | |
| 675 | # copy files in case they are not already there |
| 676 | filesMergeCPass0=( |
| 39527a15 |
677 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" |
| 678 | "${batchWorkingDirectory}/OCDB.root" |
| 679 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" |
| 71353a5f |
680 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
| 681 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" |
| 682 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" |
| 683 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" |
| 684 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" |
| fc7e1e45 |
685 | ) |
| 686 | for file in ${filesMergeCPass0[*]}; do |
| 687 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 1a0e38e3 |
688 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
| fc7e1e45 |
689 | done |
| 690 | |
| 71353a5f |
691 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 692 | |
| 693 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
694 | |
| 695 | # |
| cb473870 |
696 | echo "PWD" |
| 71353a5f |
697 | /bin/ls |
| cb473870 |
698 | echo "PWD/.." |
| 71353a5f |
699 | /bin/ls ../ |
| cb473870 |
700 | |
| fc7e1e45 |
701 | |
| 702 | #merge calibration |
| 71353a5f |
703 | chmod u+x ${mergingScript} |
| fc7e1e45 |
704 | mkdir -p ./OCDB |
| 71353a5f |
705 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 706 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" |
| 707 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} |
| fc7e1e45 |
708 | fi |
| 709 | |
| 71353a5f |
710 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 711 | if [[ -n ${pretend} ]]; then |
| 0eaf38c7 |
712 | sleep ${pretendDelay} |
| fc7e1e45 |
713 | touch CalibObjects.root |
| 714 | touch ocdb.log |
| 715 | touch merge.log |
| 617f8886 |
716 | touch dcsTime.root |
| 71353a5f |
717 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ |
| 718 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ |
| 1795556a |
719 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root |
| 720 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root |
| fc7e1e45 |
721 | else |
| 1795556a |
722 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log" |
| fc7e1e45 |
723 | |
| 617f8886 |
724 | #produce the calib trees for expert QA (dcsTime.root) |
| 725 | goMakeLocalOCDBaccessConfig ./OCDB |
| 726 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 727 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 728 | fi |
| 729 | |
| fc7e1e45 |
730 | ### produce the output |
| 731 | #tar the produced OCDB for reuse |
| ef77d296 |
732 | #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB |
| 733 | |
| 734 | # Create tarball with OCDB, store on the shared directory, create signal file on batch directory |
| 735 | mkdir -p ${commonOutputPath}/meta |
| 736 | baseTar="cpass0.localOCDB.${runNumber}.tgz" |
| 737 | tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \ |
| 738 | mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \ |
| 739 | touch ${batchWorkingDirectory}/${baseTar}.done |
| fc7e1e45 |
740 | |
| 71353a5f |
741 | /bin/ls |
| 5ed7c82c |
742 | |
| 743 | #copy all to output dir |
| 13abb339 |
744 | echo "paranoidCp ${runpath}/* ${outputDir}" |
| 745 | paranoidCp ${runpath}/* ${outputDir} |
| 617f8886 |
746 | |
| 71353a5f |
747 | if [[ -n ${generateMC} ]]; then |
| 748 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done |
| 749 | fi |
| 750 | |
| fc7e1e45 |
751 | #validate merging cpass0 |
| 5ed7c82c |
752 | cd ${outputDir} |
| ef77d296 |
753 | if summarizeLogs >> ${doneFileTmp}; then |
| 754 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp} |
| 755 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp} |
| ab946185 |
756 | fi |
| fc7e1e45 |
757 | |
| 71353a5f |
758 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| ef77d296 |
759 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 760 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 5de3064b |
761 | return 0 |
| 78d0bbfd |
762 | ) |
| fc7e1e45 |
763 | |
| 764 | goMergeCPass1() |
| 78d0bbfd |
765 | ( |
| fc7e1e45 |
766 | # |
| 767 | # find the output files and merge them |
| 768 | # |
| 769 | |
| 71353a5f |
770 | outputDir=${1} |
| 771 | ocdbStorage=${2} |
| 772 | configFile=${3} |
| 773 | runNumber=${4} |
| 774 | calibrationFilesToMerge=${5} |
| 775 | qaFilesToMerge=${6} |
| 444b9916 |
776 | filteredFilesToMerge=${7} |
| 777 | shift 7 |
| 6fc48f9e |
778 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 71353a5f |
779 | |
| 39527a15 |
780 | #record the working directory provided by the batch system |
| 781 | batchWorkingDirectory=${PWD} |
| 782 | |
| 71353a5f |
783 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| ef77d296 |
784 | |
| 785 | # This file signals that everything went fine |
| 786 | doneFileBase="merge.cpass1.run${runNumber}.done" |
| 787 | |
| 788 | # We will have two copies of the file |
| 789 | mkdir -p "${commonOutputPath}/meta" || return 1 |
| 790 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" |
| 791 | doneFile="${commonOutputPath}/meta/${doneFileBase}" |
| 71353a5f |
792 | |
| 793 | umask 0002 |
| 794 | ulimit -c unlimited |
| fc7e1e45 |
795 | |
| 796 | #clean up first: |
| 71353a5f |
797 | rm -f ${outputDir}/*.log |
| 798 | rm -f ${outputDir}/*.root |
| 799 | rm -f ${outputDir}/*done |
| 800 | |
| 801 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 802 | |
| 71353a5f |
803 | runpath=${outputDir} |
| 0eaf38c7 |
804 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX) |
| e91a9c78 |
805 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber} |
| 71353a5f |
806 | |
| 807 | mkdir -p ${runpath} |
| 1795556a |
808 | if [[ ! -d ${runpath} ]]; then |
| ef77d296 |
809 | touch ${doneFileTmp} |
| 810 | echo "not able to make the runpath ${runpath}" >> ${doneFileTmp} |
| 811 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 812 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 1795556a |
813 | return 1 |
| 814 | fi |
| 0eaf38c7 |
815 | if ! cd ${runpath}; then |
| ef77d296 |
816 | touch ${doneFileTmp} |
| 817 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} |
| 818 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 819 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 0eaf38c7 |
820 | return 1 |
| 821 | fi |
| 71353a5f |
822 | |
| 823 | logOutputDir=${runpath} |
| 824 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} |
| 825 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log |
| 826 | echo "${0} $*" |
| 827 | |
| 828 | calibrationOutputFileName='AliESDfriends_v1.root' |
| 829 | qaOutputFileName='QAresults*.root' |
| fc7e1e45 |
830 | mergingScript="mergeMakeOCDB.byComponent.sh" |
| cb473870 |
831 | #important to have the string "Stage.txt" in the filename to trigger the merging |
| 832 | #it has to be a list of directories containing the files |
| fc7e1e45 |
833 | qaMergedOutputFileName="QAresults_merged.root" |
| 834 | |
| 835 | echo goMergeCPass1 SETUP: |
| 71353a5f |
836 | echo runNumber=${runNumber} |
| 837 | echo outputDir=${outputDir} |
| 838 | echo ocdbStorage=${ocdbStorage} |
| 839 | echo calibrationFilesToMerge=$calibrationFilesToMerge |
| 840 | echo qaFilesToMerge=$qaFilesToMerge |
| 841 | echo calibrationOutputFileName=${calibrationOutputFileName} |
| 842 | echo mergingScript=${mergingScript} |
| fc7e1e45 |
843 | |
| 844 | # copy files in case they are not already there |
| 845 | filesMergeCPass1=( |
| 39527a15 |
846 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" |
| 847 | "${batchWorkingDirectory}/${qaFilesToMerge}" |
| 848 | "${batchWorkingDirectory}/OCDB.root" |
| 849 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" |
| 3fbef3b9 |
850 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
| 39527a15 |
851 | "${batchWorkingDirectory}/QAtrain_duo.C" |
| 71353a5f |
852 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
| 853 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" |
| 854 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" |
| 855 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" |
| 856 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" |
| 857 | "${trustedQAtrainMacro}" |
| 858 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" |
| fc7e1e45 |
859 | ) |
| 860 | for file in ${filesMergeCPass1[*]}; do |
| 861 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 1a0e38e3 |
862 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
| fc7e1e45 |
863 | done |
| 864 | |
| 71353a5f |
865 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
| 866 | |
| cb473870 |
867 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
| 868 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then |
| 71353a5f |
869 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| 870 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
| cb473870 |
871 | else |
| 872 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" |
| 873 | fi |
| 874 | |
| 71353a5f |
875 | alirootInfo > ALICE_ROOT.log |
| fc7e1e45 |
876 | |
| 877 | # |
| 71353a5f |
878 | /bin/ls |
| fc7e1e45 |
879 | |
| 880 | #merge calibration |
| 71353a5f |
881 | chmod u+x ${mergingScript} |
| fc7e1e45 |
882 | mkdir -p OCDB |
| 444b9916 |
883 | |
| 884 | #if not provided, create the lists of files to merge |
| 885 | if [[ ! -f ${filteredFilesToMerge} ]]; then |
| 886 | echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}" |
| 887 | /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge} |
| 888 | fi |
| 71353a5f |
889 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
| 444b9916 |
890 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" |
| 891 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} |
| 892 | fi |
| 893 | if [[ ! -f ${qaFilesToMerge} ]]; then |
| 894 | #find the files, but only store the directories (QAtrain_duo.C requires this) |
| 895 | echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" |
| 896 | /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} |
| fc7e1e45 |
897 | fi |
| 898 | |
| 71353a5f |
899 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
| 900 | if [[ -n ${pretend} ]]; then |
| 0eaf38c7 |
901 | sleep ${pretendDelay} |
| fc7e1e45 |
902 | touch ocdb.log |
| 71353a5f |
903 | touch cpass1.localOCDB.${runNumber}.tgz |
| 71353a5f |
904 | touch ${qaMergedOutputFileName} |
| fc7e1e45 |
905 | touch merge.log |
| cb473870 |
906 | touch trending.root |
| 444b9916 |
907 | touch FilterEvents_Trees.root |
| ab946185 |
908 | touch CalibObjects.root |
| 909 | touch dcsTime.root |
| 910 | touch ${qaMergedOutputFileName} |
| 617f8886 |
911 | mkdir -p OCDB |
| fc7e1e45 |
912 | else |
| 617f8886 |
913 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
| 914 | |
| 915 | #merge QA (and filtered trees) |
| 916 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF |
| 917 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF |
| 918 | |
| 919 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 920 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" |
| 71353a5f |
921 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 922 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log |
| 923 | mv QAresults_barrel.root ${qaMergedOutputFileName} |
| cb473870 |
924 | mv trending_barrel.root trending.root |
| 444b9916 |
925 | |
| 926 | #merge filtered trees |
| 927 | echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
| 922d0b10 |
928 | aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log |
| 617f8886 |
929 | |
| 930 | #produce the calib trees for expert QA |
| 931 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" |
| 922d0b10 |
932 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log |
| fc7e1e45 |
933 | fi |
| 617f8886 |
934 | |
| ef77d296 |
935 | # Create tarball with OCDB, store on the shared directory, create signal file on batch directory |
| 936 | mkdir -p ${commonOutputPath}/meta |
| 937 | baseTar="cpass1.localOCDB.${runNumber}.tgz" |
| 938 | tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \ |
| 939 | mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \ |
| 940 | touch ${batchWorkingDirectory}/${baseTar}.done |
| 71353a5f |
941 | |
| 942 | /bin/ls |
| 5ed7c82c |
943 | |
| 944 | #copy all to output dir |
| 13abb339 |
945 | echo "paranoidCp ${runpath}/* ${outputDir}" |
| 946 | paranoidCp ${runpath}/* ${outputDir} |
| 5ed7c82c |
947 | |
| fc7e1e45 |
948 | #validate merge cpass1 |
| 5ed7c82c |
949 | cd ${outputDir} |
| ef77d296 |
950 | if summarizeLogs >> ${doneFileTmp}; then |
| 951 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp} |
| 952 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp} |
| 953 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp} |
| 954 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp} |
| 955 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} |
| 922d0b10 |
956 | else |
| ef77d296 |
957 | if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then |
| 958 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp} |
| 922d0b10 |
959 | fi |
| ef77d296 |
960 | if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then |
| 961 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} |
| 922d0b10 |
962 | fi |
| ab946185 |
963 | fi |
| 71353a5f |
964 | |
| 965 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| ef77d296 |
966 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 967 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| 5de3064b |
968 | return 0 |
| 78d0bbfd |
969 | ) |
| cb473870 |
970 | |
| 971 | goMerge() |
| 78d0bbfd |
972 | ( |
| cb473870 |
973 | #generic root merge using CPass1 merge.C script |
| 71353a5f |
974 | inputList=${1} |
| 975 | outputFile=${2} |
| 0eaf38c7 |
976 | configFile=${3-"benchmark.config"} |
| 71353a5f |
977 | shift 3 |
| 6fc48f9e |
978 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 39527a15 |
979 | |
| 980 | #record the working directory provided by the batch system |
| 981 | batchWorkingDirectory=${PWD} |
| fc7e1e45 |
982 | |
| 71353a5f |
983 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 |
| 984 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 |
| 985 | umask 0002 |
| 986 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 987 | rm -f ${outputFile} |
| 988 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log |
| 5de3064b |
989 | return 0 |
| 78d0bbfd |
990 | ) |
| fc7e1e45 |
991 | |
| cb473870 |
992 | goSubmitMakeflow() |
| 5dfac970 |
993 | { |
| cb473870 |
994 | #run |
| 71353a5f |
995 | productionID=${1} |
| 996 | inputFileList=${2} |
| 997 | configFile=${3} |
| 998 | shift 3 |
| 444b9916 |
999 | extraOpts=("$@") |
| 6fc48f9e |
1000 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
| 39527a15 |
1001 | |
| 1002 | #record the working directory provided by the batch system |
| 1003 | batchWorkingDirectory=${PWD} |
| fc7e1e45 |
1004 | |
| 5dfac970 |
1005 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 1006 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
1007 | |
| 71353a5f |
1008 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then |
| cb473870 |
1009 | echo "setting the makflow path from the config: " |
| 1010 | echo " export PATH=${makeflowPath}:${PATH}" |
| 1011 | export PATH=${makeflowPath}:${PATH} |
| 1012 | fi |
| 1013 | |
| 39527a15 |
1014 | #create the common output dir and the meta dir |
| 1015 | commonOutputPath=${baseOutputDirectory}/${productionID} |
| 1016 | if [[ -d ${commonOutputPath} ]]; then |
| 1017 | echo "output dir ${commonOutputPath} exists!" |
| 0eaf38c7 |
1018 | #return 1 |
| 39527a15 |
1019 | else |
| 1020 | mkdir -p ${commonOutputPath} |
| 1021 | fi |
| 1022 | mkdir -p ${commonOutputPath}/meta |
| 839b07ca |
1023 | |
| 0eaf38c7 |
1024 | self=${0} |
| 1025 | #if which greadlink; then self=$(greadlink -f "${0}"); fi |
| 839b07ca |
1026 | |
| 0eaf38c7 |
1027 | #for reference copy the setup to the output dir |
| 839b07ca |
1028 | cp ${self} ${commonOutputPath} |
| 1029 | cp ${configFile} ${commonOutputPath} |
| 333bc229 |
1030 | cp ${inputFileList} ${commonOutputPath} |
| 39527a15 |
1031 | |
| cb473870 |
1032 | #submit - use makeflow if available, fall back to old stuff when makeflow not there |
| 1033 | if which makeflow; then |
| 39527a15 |
1034 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow |
| 1795556a |
1035 | cp benchmark.makeflow ${commonOutputPath} |
| cb473870 |
1036 | makeflow ${makeflowOptions} benchmark.makeflow |
| 343130c8 |
1037 | makeflow_exitcode=$? |
| cb473870 |
1038 | else |
| 1039 | echo "no makeflow!" |
| 343130c8 |
1040 | return 1 |
| cb473870 |
1041 | fi |
| 054fdec5 |
1042 | |
| 1043 | #summarize the run based on the makeflow log |
| 1044 | #and add it to the end of summary log |
| 1045 | awk '/STARTED/ {startTime=$3} |
| 1046 | /COMPLETED/ {endTime=$3} |
| 1047 | END {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \ |
| 1048 | benchmark.makeflow.makeflowlog | tee -a summary.log |
| 1049 | paranoidCp summary.log ${commonOutputPath} |
| 1050 | |
| 343130c8 |
1051 | return $makeflow_exitcode |
| 5dfac970 |
1052 | } |
| fc7e1e45 |
1053 | |
| 5dfac970 |
1054 | goGenerateMakeflow() |
| 78d0bbfd |
1055 | ( |
| 5dfac970 |
1056 | #generate the makeflow file |
| ab946185 |
1057 | [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1 |
| 71353a5f |
1058 | productionID=${1} |
| 1059 | inputFileList=${2} |
| 1060 | configFile=${3} |
| 1061 | shift 3 |
| 444b9916 |
1062 | extraOpts=("$@") |
| 0eaf38c7 |
1063 | |
| 1064 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them |
| 1065 | for (( i=0;i<${#extraOpts[@]};i++ )); do |
| 1066 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") |
| 1067 | done |
| 1068 | extraOpts+=("encodedSpaces=1") |
| 1069 | |
| 39527a15 |
1070 | if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi |
| 1071 | |
| 1a0e38e3 |
1072 | #extra safety |
| 1073 | if [[ -z ${commonOutputPath} ]]; then |
| 1074 | commonOutputPath=${baseOutputDirectory}/${productionID} |
| 1075 | extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" ) |
| 1076 | fi |
| bff71da0 |
1077 | |
| 39527a15 |
1078 | #record the working directory provided by the batch system |
| 1079 | batchWorkingDirectory=${PWD} |
| fc7e1e45 |
1080 | |
| 5dfac970 |
1081 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
| 1082 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 |
| fc7e1e45 |
1083 | |
| 5dfac970 |
1084 | #these files will be made a dependency - will be copied to the working dir of the jobs |
| 1085 | declare -a copyFiles |
| 1086 | inputFiles=( |
| 1087 | "OCDB.root" |
| 1088 | "localOCDBaccessConfig.C" |
| cb473870 |
1089 | "QAtrain_duo.C" |
| 1090 | "runCPass1.sh" |
| 1091 | "recCPass1.C" |
| 1092 | "recCPass1_OuterDet.C" |
| 1093 | "runCalibTrain.C" |
| 1094 | "runCPass0.sh" |
| 1095 | "recCPass0.C" |
| ef77d296 |
1096 | "runQA.sh" |
| 5dfac970 |
1097 | ) |
| 1098 | for file in ${inputFiles[*]}; do |
| 1099 | [[ -f ${file} ]] && copyFiles+=("${file}") |
| 1100 | done |
| 1101 | |
| 1102 | #create the makeflow file |
| cb473870 |
1103 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" |
| 444b9916 |
1104 | declare -A arr_cpass0_merged arr_cpass1_merged |
| 1105 | declare -A arr_cpass0_calib_list arr_cpass1_calib_list |
| 1106 | declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list |
| 6fc48f9e |
1107 | declare -A arr_cpass0_profiled_outputs |
| 444b9916 |
1108 | declare -A listOfRuns |
| 1109 | [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1 |
| 617f8886 |
1110 | while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList} |
| 444b9916 |
1111 | for runNumber in "${!listOfRuns[@]}"; do |
| 71353a5f |
1112 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
1113 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| fc7e1e45 |
1114 | |
| 444b9916 |
1115 | unset arr_cpass0_outputs |
| 1116 | unset arr_cpass1_outputs |
| 1117 | declare -a arr_cpass0_outputs |
| 1118 | declare -a arr_cpass1_outputs |
| fc7e1e45 |
1119 | |
| ef77d296 |
1120 | #Header |
| 1121 | echo "### Automatically generated on $(LANG=C date) ###" |
| 1122 | echo ; echo |
| 1123 | |
| cb473870 |
1124 | jobindex=0 |
| 6fc48f9e |
1125 | inputFile="" |
| 5dfac970 |
1126 | while read inputFile; do |
| 1127 | currentDefaultOCDB=${defaultOCDB} |
| cb473870 |
1128 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 1129 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
1130 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) |
| cb473870 |
1131 | fi |
| ef77d296 |
1132 | guessRunData ${inputFile} |
| 1133 | |
| 1134 | #Set variables |
| 1135 | echo "### Variables ###" |
| 1136 | echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\"" |
| 1137 | echo ; echo |
| fc7e1e45 |
1138 | |
| 5dfac970 |
1139 | #CPass0 |
| ef77d296 |
1140 | #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" |
| 1141 | arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done" |
| 1142 | echo "### CPass0 ###" |
| 1143 | echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}" |
| 1144 | echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " |
| 1145 | echo ; echo |
| 5dfac970 |
1146 | |
| 1147 | #CPass1 |
| ef77d296 |
1148 | #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" |
| 1149 | arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done" |
| 1150 | echo "### CPass1 ###" |
| 1151 | echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}" |
| 1152 | echo " ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " |
| 1153 | echo ; echo |
| 5dfac970 |
1154 | ((jobindex++)) |
| fc7e1e45 |
1155 | |
| 71353a5f |
1156 | done< <(grep "/000${runNumber}/" ${inputFileList}) |
| cb473870 |
1157 | |
| 5dfac970 |
1158 | #CPass0 list of Calib files to merge |
| ef77d296 |
1159 | #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list" |
| 1160 | arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list" |
| 1161 | echo "### Produces the list of CPass0 files to merge (executes locally) ###" |
| 1162 | echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}" |
| 1163 | echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}" |
| 1164 | echo ; echo |
| 71353a5f |
1165 | |
| 1166 | #CPass0 merging |
| ef77d296 |
1167 | echo "### Merges CPass0 files ###" |
| 1168 | #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" |
| 1169 | arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done" |
| 1170 | echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}" |
| 1171 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" " |
| 1172 | echo ; echo |
| fc7e1e45 |
1173 | |
| ab946185 |
1174 | #CPass1 list of Calib/QA/ESD/filtered files |
| 1175 | # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger |
| 1176 | # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid |
| 8d6a0a43 |
1177 | # this corresponds to the last merging stage) |
| ef77d296 |
1178 | #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" |
| 1179 | arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list" |
| 1180 | echo "### Lists CPass1 QA ###" |
| 444b9916 |
1181 | echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" |
| ef77d296 |
1182 | echo " LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}" |
| 1183 | echo ; echo |
| 1184 | |
| 1185 | #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list" |
| 1186 | arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list" |
| 1187 | echo "### Lists CPass1 Calib ###" |
| 1188 | echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" |
| 1189 | echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}" |
| 1190 | echo ; echo |
| 1191 | |
| 1192 | #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list" |
| 1193 | arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list" |
| 1194 | echo "### Lists CPass1 ESDs ###" |
| 1195 | echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" |
| 1196 | echo " LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}" |
| 1197 | echo ; echo |
| 1198 | |
| 1199 | #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list" |
| 1200 | arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list" |
| 1201 | echo "### Lists CPass1 filtered ###" |
| 1202 | echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" |
| 1203 | echo " LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}" |
| 1204 | echo ; echo |
| 444b9916 |
1205 | |
| 71353a5f |
1206 | #CPass1 merging |
| ef77d296 |
1207 | #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" |
| 1208 | arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done" |
| 1209 | echo "### Merges CPass1 files ###" |
| 1210 | echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}" |
| 1211 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" " |
| 1212 | echo ; echo |
| fc7e1e45 |
1213 | |
| 6fc48f9e |
1214 | #CPass0 wrapped in a profiling tool (valgrind,....) |
| 39527a15 |
1215 | if [[ -n ${profilingCommand} ]]; then |
| 0eaf38c7 |
1216 | inputFile=$(grep -m1 "${runNumber}/" ${inputFileList}) |
| 1217 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 |
| 1218 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) |
| 1219 | jobindex="profiling" |
| 1220 | |
| ef77d296 |
1221 | #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done" |
| 1222 | arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done" |
| 1223 | echo "### CPass0 in a profiler ###" |
| 1224 | echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}" |
| 0eaf38c7 |
1225 | profilingCommand=$(encSpaces "${profilingCommand}") |
| ef77d296 |
1226 | echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}" |
| 1227 | echo ; echo |
| 6fc48f9e |
1228 | fi |
| 1229 | |
| 444b9916 |
1230 | done #runs |
| fc7e1e45 |
1231 | |
| 5dfac970 |
1232 | #Summary |
| ef77d296 |
1233 | echo "### Summary ###" |
| 1234 | echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}" |
| 1235 | echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}" |
| 1236 | echo ; echo |
| cb473870 |
1237 | |
| 5de3064b |
1238 | return 0 |
| 78d0bbfd |
1239 | ) |
| cb473870 |
1240 | |
| 1241 | goPrintValues() |
| 78d0bbfd |
1242 | ( |
| cb473870 |
1243 | #print the values given the key from any number of files (store in output file on second argument) |
| 71353a5f |
1244 | if [[ $# -lt 3 ]]; then |
| 1245 | echo "goPrintValues key outputFile inputFiles" |
| ef77d296 |
1246 | echo "if outputFile is \"-\" don't redirect to a file" |
| 71353a5f |
1247 | return |
| 1248 | fi |
| 1249 | key=${1} |
| 1250 | outputFile=${2} |
| 1251 | [[ ${outputFile} =~ "-" ]] && outputFile="" |
| cb473870 |
1252 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk |
| 71353a5f |
1253 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} |
| 5de3064b |
1254 | return 0 |
| 78d0bbfd |
1255 | ) |
| fc7e1e45 |
1256 | |
| 5dfac970 |
1257 | goCreateQAplots() |
| 78d0bbfd |
1258 | ( |
| 5dfac970 |
1259 | umask 0002 |
| 71353a5f |
1260 | mergedQAfileList=${1} |
| 1261 | productionID=${2} |
| 1262 | outputDir=${3} |
| 1263 | configFile=${4} |
| 1264 | shift 4 |
| e48b4924 |
1265 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 71353a5f |
1266 | |
| 39527a15 |
1267 | #record the working directory provided by the batch system |
| 1268 | batchWorkingDirectory=${PWD} |
| 1269 | |
| 71353a5f |
1270 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| fc7e1e45 |
1271 | |
| 71353a5f |
1272 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} |
| 0eaf38c7 |
1273 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log |
| 71353a5f |
1274 | echo "${0} $*" |
| 1275 | |
| 1276 | olddir=${PWD} |
| 1277 | mkdir -p ${outputDir} |
| 1278 | cd ${outputDir} |
| 839b07ca |
1279 | [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1 |
| fc7e1e45 |
1280 | |
| ef77d296 |
1281 | inputFiles=( |
| 1282 | "${batchWorkingDirectory}/runQA.sh" |
| 1283 | "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh" |
| 1284 | ) |
| 1285 | for file in ${inputFiles[*]}; do |
| 1286 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
| 1287 | done |
| 1288 | |
| 1289 | echo "running QA with command:" |
| 1290 | echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}" |
| 1291 | ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}" |
| 71353a5f |
1292 | cd ${olddir} |
| 5de3064b |
1293 | return 0 |
| 78d0bbfd |
1294 | ) |
| fc7e1e45 |
1295 | |
| 5dfac970 |
1296 | goTest() |
| 78d0bbfd |
1297 | ( |
| 1298 | echo AA |
| 1299 | ) |
| fc7e1e45 |
1300 | |
| 5dfac970 |
1301 | alirootInfo() |
| 78d0bbfd |
1302 | ( |
| 5dfac970 |
1303 | umask 0002 |
| 71353a5f |
1304 | # save aliroot repository info |
| 1305 | [[ -z "${ALICE_ROOT}" ]] && return 1 |
| 1306 | |
| 1307 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" |
| 1308 | echo "\${ROOTSYS}=${ROOTSYS}" |
| 1309 | echo "\${PATH}=${PATH}" |
| 1310 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" |
| 0eaf38c7 |
1311 | echo |
| 71353a5f |
1312 | |
| 1313 | pushd ${PWD} |
| 1314 | cd ${ALICE_ROOT} |
| 71353a5f |
1315 | |
| 1316 | currentBranch=$(git rev-parse --abbrev-ref HEAD) |
| 1317 | git status |
| 5dfac970 |
1318 | echo "" |
| 1319 | echo "" |
| 71353a5f |
1320 | git diff ${currentBranch} |
| 1321 | popd |
| 5de3064b |
1322 | return 0 |
| 78d0bbfd |
1323 | ) |
| fc7e1e45 |
1324 | |
| 5dfac970 |
1325 | setYear() |
| 78d0bbfd |
1326 | ( |
| 5dfac970 |
1327 | #set the year |
| 71353a5f |
1328 | # ${1} - year to be set |
| 1329 | # ${2} - where to set the year |
| 1330 | year1=$(guessYear ${1}) |
| 1331 | year2=$(guessYear ${2}) |
| 1332 | local path=${2} |
| 1333 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} |
| 1334 | echo ${path} |
| 5de3064b |
1335 | return 0 |
| 78d0bbfd |
1336 | ) |
| fc7e1e45 |
1337 | |
| 5dfac970 |
1338 | guessPeriod() |
| 78d0bbfd |
1339 | ( |
| 5dfac970 |
1340 | #guess the period from the path, pick the rightmost one |
| 1341 | local IFS="/" |
| 71353a5f |
1342 | declare -a path=( ${1} ) |
| 5dfac970 |
1343 | local dirDepth=${#path[*]} |
| 1344 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1345 | local field=${path[${x}]} |
| 71353a5f |
1346 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break |
| 5dfac970 |
1347 | done |
| 71353a5f |
1348 | echo ${period} |
| 5de3064b |
1349 | return 0 |
| 78d0bbfd |
1350 | ) |
| fc7e1e45 |
1351 | |
| 5dfac970 |
1352 | guessYear() |
| 78d0bbfd |
1353 | ( |
| 5dfac970 |
1354 | #guess the year from the path, pick the rightmost one |
| 1355 | local IFS="/" |
| 71353a5f |
1356 | declare -a path=( ${1} ) |
| 5dfac970 |
1357 | local dirDepth=${#path[*]} |
| 1358 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1359 | local field=${path[${x}]} |
| 71353a5f |
1360 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break |
| 5dfac970 |
1361 | done |
| 71353a5f |
1362 | echo ${year} |
| 5de3064b |
1363 | return 0 |
| 78d0bbfd |
1364 | ) |
| fc7e1e45 |
1365 | |
| 5dfac970 |
1366 | guessRunNumber() |
| 78d0bbfd |
1367 | ( |
| 5dfac970 |
1368 | #guess the run number from the path, pick the rightmost one |
| 71353a5f |
1369 | #works for /path/foo/000123456/bar/... |
| 1370 | #and /path/foo.run123456.bar |
| 1371 | local IFS="/." |
| 1372 | declare -a path=( ${1} ) |
| 5dfac970 |
1373 | local dirDepth=${#path[*]} |
| 1374 | for ((x=${dirDepth}-1;x>=0;x--)); do |
| 1375 | local field=${path[${x}]} |
| 71353a5f |
1376 | field=${field/run/000} |
| 617f8886 |
1377 | [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break |
| 5dfac970 |
1378 | done |
| 71353a5f |
1379 | echo ${runNumber} |
| 5de3064b |
1380 | return 0 |
| 78d0bbfd |
1381 | ) |
| fc7e1e45 |
1382 | |
| cb473870 |
1383 | validateLog() |
| 78d0bbfd |
1384 | ( |
| 71353a5f |
1385 | log=${1} |
| cb473870 |
1386 | errorConditions=( |
| 71353a5f |
1387 | 'There was a crash' |
| 1388 | 'floating' |
| 1389 | 'error while loading shared libraries' |
| 1390 | 'std::bad_alloc' |
| 1391 | 's_err_syswatch_' |
| 1392 | 'Thread [0-9]* (Thread' |
| 1393 | 'AliFatal' |
| 1394 | 'core dumped' |
| 1395 | '\.C.*error:.*\.h: No such file' |
| 839b07ca |
1396 | 'line.*Aborted' |
| 71353a5f |
1397 | ) |
| 1398 | |
| 1399 | warningConditions=( |
| 1400 | 'This is serious !' |
| 1401 | 'rocVoltage out of range:' |
| cb473870 |
1402 | ) |
| 1403 | |
| 71353a5f |
1404 | local logstatus=0 |
| 1405 | local errorSummary="" |
| 1406 | local warningSummary="" |
| 1407 | |
| 1408 | for ((i=0; i<${#errorConditions[@]};i++)); do |
| 1409 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) |
| 1410 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1411 | errorSummary+=${tmp} |
| 1412 | done |
| 1413 | |
| 1414 | for ((i=0; i<${#warningConditions[@]};i++)); do |
| 1415 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) |
| 1416 | [[ -n ${tmp} ]] && tmp+=" : " |
| 1417 | warningSummary+=${tmp} |
| 1418 | done |
| 1419 | |
| 1420 | if [[ -n ${errorSummary} ]]; then |
| 1421 | echo "${errorSummary}" |
| 1422 | return 1 |
| 1423 | fi |
| cb473870 |
1424 | |
| 71353a5f |
1425 | if [[ -n ${warningSummary} ]]; then |
| 1426 | echo "${warningSummary}" |
| 1427 | return 2 |
| 1428 | fi |
| 1429 | |
| 1430 | return 0 |
| 78d0bbfd |
1431 | ) |
| cb473870 |
1432 | |
| 5dfac970 |
1433 | summarizeLogs() |
| 78d0bbfd |
1434 | ( |
| 5dfac970 |
1435 | #print a summary of logs |
| 1436 | logFiles=( |
| 1437 | "*.log" |
| cb473870 |
1438 | "stdout" |
| 1439 | "stderr" |
| 5dfac970 |
1440 | ) |
| fc7e1e45 |
1441 | |
| 9c124791 |
1442 | #put dir information in the output |
| 1443 | echo "dir $PWD" |
| 1444 | |
| cb473870 |
1445 | #check logs |
| 71353a5f |
1446 | local logstatus=0 |
| 5dfac970 |
1447 | for log in ${logFiles[*]}; do |
| 71353a5f |
1448 | finallog=${PWD%/}/${log} |
| 1449 | [[ ! -f ${log} ]] && continue |
| 1450 | errorSummary=$(validateLog ${log}) |
| 1451 | validationStatus=$? |
| 1452 | if [[ ${validationStatus} -eq 0 ]]; then |
| 5dfac970 |
1453 | #in pretend mode randomly report an error in rec.log some cases |
| 71353a5f |
1454 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then |
| 617f8886 |
1455 | #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" |
| 1456 | echo "${finallog} OK" |
| fc7e1e45 |
1457 | else |
| 71353a5f |
1458 | echo "${finallog} OK" |
| fc7e1e45 |
1459 | fi |
| 71353a5f |
1460 | elif [[ ${validationStatus} -eq 1 ]]; then |
| 1461 | echo "${finallog} BAD ${errorSummary}" |
| ab946185 |
1462 | logstatus=1 |
| 71353a5f |
1463 | elif [[ ${validationStatus} -eq 2 ]]; then |
| 1464 | echo "${finallog} OK MWAH ${errorSummary}" |
| 5dfac970 |
1465 | fi |
| 1466 | done |
| cb473870 |
1467 | |
| 1468 | #report core files |
| 71353a5f |
1469 | while read x; do |
| 1470 | echo ${x} |
| 1471 | chmod 644 ${x} |
| 1472 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log |
| 1473 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) |
| cb473870 |
1474 | |
| 71353a5f |
1475 | return ${logstatus} |
| 78d0bbfd |
1476 | ) |
| fc7e1e45 |
1477 | |
| 5dfac970 |
1478 | spitOutLocalOCDBaccessConfig() |
| 1479 | { |
| 1480 | umask 0002 |
| 71353a5f |
1481 | #find ${1} -name "*root" | \ |
| 1482 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ |
| 5dfac970 |
1483 | while read line |
| 1484 | do |
| 71353a5f |
1485 | local tmp=${line#${1}} |
| 5dfac970 |
1486 | echo ${tmp%/*} | \ |
| 71353a5f |
1487 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' |
| 5dfac970 |
1488 | done |
| 5de3064b |
1489 | return 0 |
| 5dfac970 |
1490 | } |
| fc7e1e45 |
1491 | |
| 71353a5f |
1492 | goMakeLocalOCDBaccessConfig() |
| 5dfac970 |
1493 | { |
| 1494 | umask 0002 |
| 1495 | # make a script that sets the specific storages form all the root files produced by CPass0 |
| 71353a5f |
1496 | local localOCDBpathCPass0=${1} |
| 1497 | local OCDBpathPrefix=${2} |
| 1498 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." |
| 1499 | |
| 1500 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then |
| 1501 | tar xzf ${localOCDBpathCPass0} |
| 1502 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" |
| 5dfac970 |
1503 | fi |
| fc7e1e45 |
1504 | |
| 5dfac970 |
1505 | echo |
| 1506 | echo creating the specific storage script |
| 1507 | echo localOCDBaccessConfig.C |
| 71353a5f |
1508 | echo based on OCDB: ${localOCDBaccessConfig} |
| 5dfac970 |
1509 | echo |
| fc7e1e45 |
1510 | |
| 5dfac970 |
1511 | local tempLocalOCDB="" |
| 1512 | if [[ -f localOCDBaccessConfig.C ]]; then |
| 0eaf38c7 |
1513 | tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX) |
| 71353a5f |
1514 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" |
| 1515 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} |
| 5dfac970 |
1516 | fi |
| fc7e1e45 |
1517 | |
| 5dfac970 |
1518 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
| 1519 | echo "{" >> localOCDBaccessConfig.C |
| 1520 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C |
| 71353a5f |
1521 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C |
| 1522 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C |
| 5dfac970 |
1523 | echo "}" >> localOCDBaccessConfig.C |
| fc7e1e45 |
1524 | |
| 71353a5f |
1525 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} |
| fc7e1e45 |
1526 | |
| 5dfac970 |
1527 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
| 1528 | echo |
| 1529 | echo "!!!!!!! CPass0 produced no OCDB entries" |
| 1530 | return 1 |
| 1531 | fi |
| 5de3064b |
1532 | return 0 |
| 5dfac970 |
1533 | } |
| fc7e1e45 |
1534 | |
| cb473870 |
1535 | goMakeFilteredTrees() |
| 78d0bbfd |
1536 | ( |
| 71353a5f |
1537 | outputDir=${1} |
| 1538 | runNumber=${2} |
| cb473870 |
1539 | #get path to input list |
| 71353a5f |
1540 | inputListfiles=${3} |
| cb473870 |
1541 | #get scale number for tracks |
| 71353a5f |
1542 | filterT=${4} |
| cb473870 |
1543 | #get scale number for V0s |
| 71353a5f |
1544 | filterV=${5} |
| cb473870 |
1545 | #get OCDB path (optional) |
| 1546 | OCDBpath=${6} |
| 1547 | #get max number of files |
| 1548 | maxFiles=${7-"1000000"} |
| 1549 | #get offset of first file |
| 1550 | offsetFile=${8-"0"} |
| 1551 | #get max number of events |
| 1552 | maxEvents=${9-"30000000"} |
| 1553 | #get offset of first event |
| 1554 | offsetEvent=${10-"0"} |
| 1555 | configFile=${11-"benchmark.config"} |
| 1556 | esdFileName=${12-"AliESDs_Barrel.root"} |
| 71353a5f |
1557 | shift 12 |
| 6fc48f9e |
1558 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
| 39527a15 |
1559 | |
| 1560 | #record the working directory provided by the batch system |
| 1561 | batchWorkingDirectory=${PWD} |
| 1562 | |
| 78d0bbfd |
1563 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| ef77d296 |
1564 | doneFileBase=filtering.cpass1.run${runNumber}.done |
| 1565 | doneFileTmp=${batchWorkingDirectory}/${doneFileBase} |
| 1566 | doneFile=${commonOutputPath}/meta/${doneFileBase} |
| 71353a5f |
1567 | |
| cb473870 |
1568 | cat > filtering.log << EOF |
| 1569 | goMakeFilteredTrees config: |
| 71353a5f |
1570 | runpath=${runpath} |
| 1571 | outputDir=${outputDir} |
| 1572 | commonOutputPath=${commonOutputPath} |
| 1573 | ALICE_ROOT=${ALICE_ROOT} |
| 1574 | PATH=${PATH} |
| 1575 | offsetEvent=$offsetEvent |
| 1576 | configFile=$configFile |
| 1577 | esdFileName=$esdFileName |
| fab31805 |
1578 | inputListfiles=$inputListfiles |
| 1579 | doneFile=$doneFile |
| cb473870 |
1580 | EOF |
| 1581 | |
| fab31805 |
1582 | #runpath=${outputDir} |
| 1583 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} |
| 1584 | #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX) |
| 1585 | #mkdir -p ${outputDir} |
| 1586 | #mkdir -p ${runpath} |
| 1587 | #if ! cd ${runpath}; then |
| 1588 | # echo "PWD=$PWD is not the runpath=${runpath}" |
| 1589 | # touch ${doneFile} |
| 1590 | # return 1 |
| 1591 | #fi |
| 1592 | |
| cb473870 |
1593 | if [[ -z ${pretend} ]];then |
| 71353a5f |
1594 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log |
| cb473870 |
1595 | else |
| 0eaf38c7 |
1596 | sleep ${pretendDelay} |
| 71353a5f |
1597 | touch filtering.log FilterEvents_Trees.root |
| cb473870 |
1598 | fi |
| 1599 | pwd |
| 71353a5f |
1600 | /bin/ls |
| 71353a5f |
1601 | summarizeLogs >> ${doneFile} |
| fab31805 |
1602 | |
| 1603 | #echo mv -f * ${outputDir} |
| 1604 | #mv -f * ${outputDir} |
| 1605 | #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} |
| 1606 | #cd ${commonOutputPath} |
| 1607 | #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
| e344162c |
1608 | |
| ef77d296 |
1609 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
| 1610 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} |
| e344162c |
1611 | |
| 5de3064b |
1612 | return 0 |
| 78d0bbfd |
1613 | ) |
| cb473870 |
1614 | |
| 1615 | submit() |
| 1616 | { |
| 1617 | umask 0002 |
| 444b9916 |
1618 | [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1 |
| 71353a5f |
1619 | JobID=${1} |
| 1620 | startID=${2} |
| 1621 | endID=${3} |
| 1622 | waitForJOBID=${4} |
| 1623 | command=${5} |
| 444b9916 |
1624 | shift 5 |
| 78d0bbfd |
1625 | local commandArgs=("$@") |
| 444b9916 |
1626 | |
| 1627 | #add quote strings around the extra arguments |
| 1628 | for ((i=0; i<${#commandArgs[@]}; i++)); do |
| 1629 | commandArgs[i]=\"${commandArgs[i]}\" |
| 1630 | done |
| cb473870 |
1631 | |
| 71353a5f |
1632 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 |
| cb473870 |
1633 | |
| 1634 | newFarm=$(which qsub|grep "^/usr/bin/qsub") |
| 1635 | |
| 1636 | batchSystem="SGE" |
| 1637 | |
| 71353a5f |
1638 | if [[ -z "${newFarm}" ]] |
| cb473870 |
1639 | then |
| 1640 | #old LSF |
| 1641 | # submit it (as job array) |
| 71353a5f |
1642 | nFiles=$(( ${endID}-${startID}+1 )) |
| 1643 | while [ ${startID} -le ${nFiles} ] ; do |
| 1644 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then |
| 1645 | endID=$(expr ${startID} + 999) |
| cb473870 |
1646 | else |
| 71353a5f |
1647 | endID=${nFiles} |
| cb473870 |
1648 | fi |
| 1649 | if [[ ${waitForJOBID} -eq 0 ]]; then |
| 71353a5f |
1650 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1651 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1652 | else |
| 71353a5f |
1653 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| 1654 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
| cb473870 |
1655 | fi |
| 71353a5f |
1656 | startID=$(expr ${endID} + 1) |
| cb473870 |
1657 | done |
| 1658 | else |
| 1659 | #new SGE farm |
| 1660 | if [[ ${waitForJOBID} =~ "000" ]]; then |
| 444b9916 |
1661 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
| 1662 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
| cb473870 |
1663 | else |
| 444b9916 |
1664 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
| 1665 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
| cb473870 |
1666 | fi |
| 1667 | fi |
| 5de3064b |
1668 | return 0 |
| cb473870 |
1669 | } |
| 1670 | |
| 1671 | goSubmitBatch() |
| 5dfac970 |
1672 | { |
| 71353a5f |
1673 | if [[ $# -lt 3 ]]; then |
| 1674 | echo "minimal use:" |
| 1675 | echo " ${0} submit fileList productionID configFile" |
| 1676 | return 0 |
| 1677 | fi |
| 1678 | |
| 1679 | productionID=${1} |
| 1680 | inputList=${2} |
| 1681 | configFile=${3:-"benchmark.config"} |
| 0eaf38c7 |
1682 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi |
| 71353a5f |
1683 | shift 3 |
| 444b9916 |
1684 | extraOpts=("$@") |
| 6fc48f9e |
1685 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
| 39527a15 |
1686 | |
| b0e401b4 |
1687 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them |
| 1688 | for (( i=0;i<${#extraOpts[@]};i++ )); do |
| 1689 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") |
| 1690 | done |
| 1691 | extraOpts+=("encodedSpaces=1") |
| ef77d296 |
1692 | #this removes the copy of the done file used by makeflow (in the running dir) |
| 1693 | extraOpts+=("removeTMPdoneFile=1") |
| b0e401b4 |
1694 | |
| 39527a15 |
1695 | #record the working directory provided by the batch system |
| 1696 | batchWorkingDirectory=${PWD} |
| fc7e1e45 |
1697 | |
| 5dfac970 |
1698 | #redirect all output to submit.log |
| 1699 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" |
| 1700 | exec 7>&1 |
| 1701 | exec 1>submit_${productionID//"/"/_}.log 2>&1 |
| fc7e1e45 |
1702 | |
| 5dfac970 |
1703 | umask 0002 |
| 71353a5f |
1704 | echo ${0}" submit $*" |
| 1705 | if [[ -z "${inputList}" || -z "${productionID}" ]] |
| 5dfac970 |
1706 | then |
| fc7e1e45 |
1707 | echo |
| 71353a5f |
1708 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" |
| fc7e1e45 |
1709 | echo |
| 71353a5f |
1710 | return |
| fc7e1e45 |
1711 | fi |
| fc7e1e45 |
1712 | |
| 5dfac970 |
1713 | # check if config file is there |
| 71353a5f |
1714 | if [ ! -f ${configFile} ]; then |
| 1715 | echo "ERROR! Config File '${configFile}' not found" >&2 |
| 1716 | return |
| 5dfac970 |
1717 | else |
| 71353a5f |
1718 | echo "Using Config File: '${configFile}'" |
| fc7e1e45 |
1719 | fi |
| 1720 | |
| 9510565c |
1721 | [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1 |
| fc7e1e45 |
1722 | |
| 71353a5f |
1723 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there |
| 0eaf38c7 |
1724 | self=${0} |
| 1725 | #if which greadlink; then self=$(greadlink -f "${0}"); fi |
| 71353a5f |
1726 | configPath=$(dirname ${configFile}) |
| 5dfac970 |
1727 | export commonOutputPath=${baseOutputDirectory}/${productionID} |
| 71353a5f |
1728 | |
| cb473870 |
1729 | mkdir -p ${commonOutputPath} |
| 71353a5f |
1730 | mkdir -p ${commonOutputPath}/logs |
| 1731 | mkdir -p ${commonOutputPath}/meta |
| 1732 | |
| 1733 | cp ${self} ${commonOutputPath} |
| 1734 | cp ${configFile} ${commonOutputPath} |
| 1735 | cp ${inputList} ${commonOutputPath} |
| cb473870 |
1736 | self=${commonOutputPath}/${self##*/} |
| 71353a5f |
1737 | chmod u+x ${self} |
| cb473870 |
1738 | configFile=${commonOutputPath}/${configFile##*/} |
| 1739 | inputList=${commonOutputPath}/${inputList##*/} |
| fc7e1e45 |
1740 | |
| 5dfac970 |
1741 | #convert to absolut pathnames |
| 0eaf38c7 |
1742 | #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi |
| 5dfac970 |
1743 | #make list of runs |
| 71353a5f |
1744 | if [[ -z ${runNumber} ]]; then |
| 1745 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) |
| 5dfac970 |
1746 | else |
| 71353a5f |
1747 | listOfRuns=${runNumber} |
| 5dfac970 |
1748 | fi |
| fc7e1e45 |
1749 | |
| 0eaf38c7 |
1750 | #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi |
| fc7e1e45 |
1751 | |
| 5dfac970 |
1752 | echo "" |
| 1753 | echo "### BEGIN CONFIGURATION ###" |
| 1754 | echo "" |
| 1755 | echo "GENERAL:" |
| 1756 | echo "" |
| 71353a5f |
1757 | echo " productionID: ${productionID}" |
| 1758 | echo " batchCommand: ${batchCommand}" |
| 1759 | echo " batchFlags: ${batchFlags}" |
| 9510565c |
1760 | echo " alirootEnv: ${alirootEnv}" |
| 1761 | ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}' |
| 1762 | ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}' |
| 71353a5f |
1763 | echo " inputList: ${inputList}" |
| 1764 | echo " configPath: ${configPath}" |
| 1765 | echo " commonOutputPath: ${commonOutputPath}" |
| 1766 | echo " defaultOCDB: ${defaultOCDB}" |
| 1767 | echo " autoOCDB: ${autoOCDB}" |
| 1768 | echo " recoTriggerOptions: ${recoTriggerOptions}" |
| 5dfac970 |
1769 | echo " runs:" |
| 1770 | echo " ${listOfRuns[*]}" |
| 1771 | echo "" |
| 1772 | echo "THE TRAIN WILL RUN:" |
| fc7e1e45 |
1773 | |
| 71353a5f |
1774 | if [ ${runCPass0reco} -eq 1 ]; then |
| 5dfac970 |
1775 | echo " Pass0 - Recontruction" |
| 1776 | fi |
| fc7e1e45 |
1777 | |
| 71353a5f |
1778 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1779 | echo " Pass0 - merging and OCDB export" |
| 1780 | fi |
| fc7e1e45 |
1781 | |
| 71353a5f |
1782 | if [ ${runCPass1reco} -eq 1 ]; then |
| 5dfac970 |
1783 | echo " Pass1 - Recontruction" |
| 1784 | fi |
| 71353a5f |
1785 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| 5dfac970 |
1786 | echo " Pass1 - merging and OCDB export" |
| 1787 | fi |
| fc7e1e45 |
1788 | |
| 5dfac970 |
1789 | echo "" |
| 1790 | echo "LIMITS:" |
| 71353a5f |
1791 | echo " max. Events/Chunk: ${nEvents}" |
| 1792 | echo " max. Number of Chunks per Run: ${nMaxChunks}" |
| 5dfac970 |
1793 | echo "" |
| 1794 | echo "### END CONFIGURATION ###" |
| 1795 | echo "" |
| fc7e1e45 |
1796 | |
| 36b5de1a |
1797 | |
| 5dfac970 |
1798 | # check if input file is there |
| 71353a5f |
1799 | if [ ! -f ${inputList} ]; then |
| 1800 | echo "ERROR! Input List '${inputList}' not found" >&2 |
| 1801 | return |
| 5dfac970 |
1802 | fi |
| 1803 | |
| 1804 | # define jobid (for dependent jobs) |
| cb473870 |
1805 | date=$(date +%Y_%m_%d_%H%M%S) |
| 5dfac970 |
1806 | #for each run we submit one jobarray: |
| fc7e1e45 |
1807 | for runNumber in ${listOfRuns[*]}; do |
| cb473870 |
1808 | |
| 71353a5f |
1809 | [[ -z ${runNumber} ]] && continue |
| fc7e1e45 |
1810 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
| cb473870 |
1811 | |
| 1812 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" |
| 1813 | JOBID1="p0_${JOBpostfix}" |
| 1814 | JOBID1wait="w0_${JOBpostfix}" |
| 1815 | JOBID2="m0_${JOBpostfix}" |
| 1816 | JOBID2wait="wm0_${JOBpostfix}" |
| 1817 | JOBID3="op0_${JOBpostfix}" |
| 1818 | JOBID3wait="wop0_${JOBpostfix}" |
| 1819 | JOBID4="p1_${JOBpostfix}" |
| 1820 | JOBID4wait="w1_${JOBpostfix}" |
| 1821 | JOBID5="m1_${JOBpostfix}" |
| 1822 | JOBID5wait="wm1_${JOBpostfix}" |
| 1823 | JOBID6="s1_${JOBpostfix}" |
| 1824 | JOBID6wait="ws1_${JOBpostfix}" |
| 1825 | JOBID7="QA_${JOBpostfix}" |
| 1826 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" |
| 1827 | JOBfilterESDcpass1="fp1_${JOBpostfix}" |
| 1828 | LASTJOB="000" |
| 1829 | |
| 0eaf38c7 |
1830 | oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList}) |
| cb473870 |
1831 | |
| 1832 | currentDefaultOCDB=${defaultOCDB} |
| 1833 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
| 1834 | if [[ ${autoOCDB} -ne 0 ]]; then |
| 71353a5f |
1835 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) |
| cb473870 |
1836 | fi |
| ef77d296 |
1837 | period=$(guessPeriod ${oneInputFile}) |
| 1838 | year=$(guessYear ${oneInputFile}) |
| cb473870 |
1839 | |
| 71353a5f |
1840 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" |
| 1841 | |
| 1842 | ############################################################################### |
| 1843 | #run one chunk with valgrind: |
| 6fc48f9e |
1844 | if [[ -n ${profilingCommand} ]]; then |
| 1845 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 |
| 1846 | [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" |
| ef77d296 |
1847 | submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}" |
| 71353a5f |
1848 | fi |
| fc7e1e45 |
1849 | |
| 5dfac970 |
1850 | ################################################################################ |
| 1851 | ################################################################################ |
| 1852 | # run the CPass0 if requested |
| fc7e1e45 |
1853 | |
| 71353a5f |
1854 | if [ ${runCPass0reco} -eq 1 ]; then |
| fc7e1e45 |
1855 | |
| fc7e1e45 |
1856 | echo |
| 71353a5f |
1857 | echo "starting CPass0... for run ${runNumber}" |
| fc7e1e45 |
1858 | echo |
| fc7e1e45 |
1859 | |
| 5dfac970 |
1860 | # create directory and copy all files that are needed |
| ef77d296 |
1861 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0" |
| 71353a5f |
1862 | mkdir -p ${targetDirectory} |
| 1863 | |
| cb473870 |
1864 | filesCPass0=( |
| 71353a5f |
1865 | "${configPath}/runCPass0.sh" |
| 1866 | "${configPath}/recCPass0.C" |
| 1867 | "${configPath}/runCalibTrain.C" |
| 1868 | "${configPath}/localOCDBaccessConfig.C" |
| 1869 | "${configPath}/OCDB*.root" |
| 1870 | "${configPath}/sim.C" |
| 1871 | "${configPath}/rec.C" |
| 1872 | "${configPath}/Config.C" |
| cb473870 |
1873 | ) |
| 1874 | for file in ${filesCPass0[*]}; do |
| 1875 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1876 | done |
| fc7e1e45 |
1877 | |
| 71353a5f |
1878 | localInputList=${targetDirectory}/${inputList##*/} |
| 1879 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 5dfac970 |
1880 | # limit nFiles to nMaxChunks |
| 71353a5f |
1881 | nFiles=$(wc -l < ${localInputList}) |
| 1882 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 |
| 1883 | echo "raw files in list: ${nFiles}" |
| 1884 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1885 | nFiles=${nMaxChunks} |
| 5dfac970 |
1886 | fi |
| 71353a5f |
1887 | echo "raw files to process: ${nFiles}" |
| 1888 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1889 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1890 | nFilesToWaitFor=${nFiles} |
| 5dfac970 |
1891 | else |
| 71353a5f |
1892 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| 5dfac970 |
1893 | fi |
| 71353a5f |
1894 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1895 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
1896 | |
| 444b9916 |
1897 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" |
| fc7e1e45 |
1898 | |
| 5dfac970 |
1899 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 0ff60de9 |
1900 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} |
| 71353a5f |
1901 | LASTJOB=${JOBID1wait} |
| fc7e1e45 |
1902 | |
| 5dfac970 |
1903 | fi #end running CPass0 |
| 1904 | ################################################################################ |
| fc7e1e45 |
1905 | |
| fc7e1e45 |
1906 | |
| 5dfac970 |
1907 | ################################################################################ |
| 1908 | # submit merging of CPass0, depends on the reconstruction |
| fc7e1e45 |
1909 | |
| 71353a5f |
1910 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
1911 | |
| 5dfac970 |
1912 | echo |
| 71353a5f |
1913 | echo "submit CPass0 merging for run ${runNumber}" |
| 5dfac970 |
1914 | echo |
| fc7e1e45 |
1915 | |
| ef77d296 |
1916 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0" |
| 71353a5f |
1917 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
1918 | |
| cb473870 |
1919 | #copy the scripts |
| 1920 | filesMergeCPass0=( |
| 71353a5f |
1921 | "${configPath}/OCDB.root" |
| 1922 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 1923 | "${configPath}/mergeMakeOCDB.sh" |
| 1924 | "${configPath}/localOCDBaccessConfig.C" |
| 1925 | "${configPath}/mergeByComponent.C" |
| 1926 | "${configPath}/makeOCDB.C" |
| 1927 | "${configPath}/merge.C" |
| cb473870 |
1928 | ) |
| 1929 | for file in ${filesMergeCPass0[*]}; do |
| 1930 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1931 | done |
| 1932 | |
| 444b9916 |
1933 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}" |
| 71353a5f |
1934 | LASTJOB=${JOBID2} |
| fc7e1e45 |
1935 | |
| 71353a5f |
1936 | if [[ -n ${generateMC} ]]; then |
| 444b9916 |
1937 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done |
| 71353a5f |
1938 | LASTJOB="mrl${JOBpostfix}" |
| 1939 | fi |
| fc7e1e45 |
1940 | |
| 5dfac970 |
1941 | echo |
| fc7e1e45 |
1942 | fi |
| 5dfac970 |
1943 | # end of merging CPass0 |
| 1944 | ################################################################################ |
| fc7e1e45 |
1945 | |
| 5dfac970 |
1946 | ################################################################################ |
| 1947 | ################################################################################ |
| 1948 | # run the CPass1 if requested |
| fc7e1e45 |
1949 | |
| 71353a5f |
1950 | if [ ${runCPass1reco} -eq 1 ]; then |
| fc7e1e45 |
1951 | |
| ef77d296 |
1952 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1" |
| 71353a5f |
1953 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done |
| fc7e1e45 |
1954 | |
| 5dfac970 |
1955 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
| 71353a5f |
1956 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" |
| fc7e1e45 |
1957 | |
| 5dfac970 |
1958 | echo |
| 71353a5f |
1959 | echo "starting CPass1... for run ${runNumber}" |
| 5dfac970 |
1960 | echo |
| 1961 | |
| 1962 | # create directory and copy all files that are needed |
| 71353a5f |
1963 | mkdir -p ${targetDirectory} |
| 5dfac970 |
1964 | |
| cb473870 |
1965 | filesCPass1=( |
| 71353a5f |
1966 | "${configPath}/runCPass1.sh" |
| 1967 | "${configPath}/recCPass1.C" |
| 1968 | "${configPath}/recCPass1_OuterDet.C" |
| 1969 | "${configPath}/runCalibTrain.C" |
| 1970 | "${configPath}/QAtrain_duo.C" |
| 1971 | "${configPath}/localOCDBaccessConfig.C" |
| 1972 | "${configPath}/OCDB.root" |
| cb473870 |
1973 | ) |
| 1974 | for file in ${filesCPass1[*]}; do |
| 1975 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 1976 | done |
| 5dfac970 |
1977 | |
| 71353a5f |
1978 | if [[ -n ${generateMC} ]]; then |
| 1979 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list |
| 1980 | else |
| 1981 | localInputList=${targetDirectory}/${inputList##*/} |
| 1982 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} |
| 1983 | fi |
| 5dfac970 |
1984 | # limit nFiles to nMaxChunks |
| 71353a5f |
1985 | nFiles=$(wc -l < ${localInputList}) |
| 1986 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue |
| 1987 | echo "raw files in list: ${nFiles}" |
| 1988 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then |
| 1989 | nFiles=${nMaxChunks} |
| 5dfac970 |
1990 | fi |
| 71353a5f |
1991 | echo "raw files to process: ${nFiles}" |
| 1992 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 |
| 1993 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then |
| 1994 | nFilesToWaitFor=${nFiles} |
| fc7e1e45 |
1995 | else |
| 71353a5f |
1996 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
| fc7e1e45 |
1997 | fi |
| 71353a5f |
1998 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
| 1999 | echo "merging will start after ${nFilesToWaitFor} jobs are done" |
| fc7e1e45 |
2000 | |
| 444b9916 |
2001 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" |
| fc7e1e45 |
2002 | |
| 5dfac970 |
2003 | ################################################################################ |
| 2004 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
| 0ff60de9 |
2005 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} |
| 71353a5f |
2006 | LASTJOB=${JOBID4wait} |
| 5dfac970 |
2007 | ################################################################################ |
| fc7e1e45 |
2008 | |
| 5dfac970 |
2009 | echo |
| 2010 | fi #end running CPass1 |
| fc7e1e45 |
2011 | |
| 5dfac970 |
2012 | ################################################################################ |
| 2013 | # submit merging of CPass1, depends on the reconstruction |
| 71353a5f |
2014 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
| fc7e1e45 |
2015 | |
| 5dfac970 |
2016 | echo |
| 71353a5f |
2017 | echo "submit CPass1 merging for run ${runNumber}" |
| 5dfac970 |
2018 | echo |
| fc7e1e45 |
2019 | |
| ef77d296 |
2020 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1" |
| 71353a5f |
2021 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done |
| 2022 | mkdir -p ${targetDirectory} |
| fc7e1e45 |
2023 | |
| cb473870 |
2024 | # copy files |
| 2025 | filesMergeCPass1=( |
| 71353a5f |
2026 | "${configPath}/OCDB.root" |
| 2027 | "${configPath}/localOCDBaccessConfig.C" |
| 2028 | "${configPath}/mergeMakeOCDB.byComponent.sh" |
| 2029 | "${configPath}/mergeByComponent.C" |
| 2030 | "${configPath}/makeOCDB.C" |
| 2031 | "${configPath}/merge.C" |
| 2032 | "${configPath}/mergeMakeOCDB.sh" |
| 2033 | "${configPath}/QAtrain_duo.C" |
| cb473870 |
2034 | ) |
| 2035 | for file in ${filesMergeCPass1[*]}; do |
| 2036 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} |
| 2037 | done |
| 2038 | |
| 444b9916 |
2039 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}" |
| 71353a5f |
2040 | LASTJOB=${JOBID5} |
| 5dfac970 |
2041 | echo |
| 2042 | fi |
| 2043 | |
| 444b9916 |
2044 | ############################### |
| 2045 | #if [ ${runESDfiltering} -eq 1 ]; then |
| 2046 | # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list |
| 2047 | # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done |
| 2048 | # echo |
| 2049 | # echo submitting filtering for run ${runNumber} |
| 2050 | # echo |
| 2051 | # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done |
| ef77d296 |
2052 | # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" |
| 444b9916 |
2053 | # LASTJOB=${JOBfilterESDcpass1} |
| 2054 | #fi |
| cb473870 |
2055 | |
| 5dfac970 |
2056 | done |
| 2057 | |
| 71353a5f |
2058 | ################################################################################# |
| 2059 | ################################################################################# |
| 444b9916 |
2060 | #if [ ${runESDfiltering} -eq 1 ]; then |
| 2061 | # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait} |
| 2062 | #else |
| 2063 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait} |
| 2064 | #fi |
| 71353a5f |
2065 | LASTJOB=${JOBID5wait} |
| cb473870 |
2066 | |
| 5de3064b |
2067 | ################################################################################# |
| 2068 | echo |
| 2069 | echo "submit make a summary" |
| 2070 | echo |
| 5dfac970 |
2071 | |
| 3d530ff9 |
2072 | [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}") |
| 4f8ecc60 |
2073 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}" |
| 5de3064b |
2074 | LASTJOB=${JOBID6} |
| 71353a5f |
2075 | ################################################################################# |
| 5dfac970 |
2076 | |
| 5dfac970 |
2077 | #restore stdout |
| 2078 | exec 1>&7 7>&- |
| 2079 | echo "jobs submitted." |
| 5de3064b |
2080 | return 0 |
| fc7e1e45 |
2081 | } |
| 2082 | |
| cb473870 |
2083 | goWaitForOutput() |
| 78d0bbfd |
2084 | ( |
| cb473870 |
2085 | umask 0002 |
| 71353a5f |
2086 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 |
| 2087 | echo searchPath=${1} |
| 2088 | echo fileName=${2} |
| 2089 | echo numberOfFiles=${3} |
| 2090 | echo maxSecondsToWait=${4} |
| 2091 | searchPath=${1} |
| 2092 | fileName=${2} |
| 2093 | numberOfFiles=${3} |
| 2094 | maxSecondsToWait=${4} |
| 0ff60de9 |
2095 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}" |
| 71353a5f |
2096 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) |
| cb473870 |
2097 | while true; do |
| cb473870 |
2098 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) |
| 71353a5f |
2099 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" |
| 2100 | [[ ${n} -ge ${numberOfFiles} ]] && break |
| 2101 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break |
| cb473870 |
2102 | sleep 60 |
| 2103 | done |
| 2104 | echo "DONE! exiting..." |
| 5de3064b |
2105 | return 0 |
| 78d0bbfd |
2106 | ) |
| cb473870 |
2107 | |
| 71353a5f |
2108 | mergeSysLogs() |
| 78d0bbfd |
2109 | ( |
| 71353a5f |
2110 | outputFile=${1} |
| 2111 | shift |
| 2112 | inputFiles="$@" |
| 2113 | i=0 |
| 2114 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi |
| 2115 | while read x; do |
| 2116 | runNumber=$(guessRunNumber ${x}) |
| 2117 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue |
| 2118 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} |
| 2119 | (( i++ )) |
| 2120 | done < <(ls -1 ${inputFiles}) > ${outputFile} |
| 5de3064b |
2121 | return 0 |
| 78d0bbfd |
2122 | ) |
| 71353a5f |
2123 | |
| 2124 | goMakeMergedSummaryTree() |
| 78d0bbfd |
2125 | ( |
| 71353a5f |
2126 | # create list of calibration entries |
| 2127 | # takes no arguments, just run it in the base output |
| 2128 | # directory with the following files in the working directory |
| 2129 | # |
| 2130 | # Calibration file lists: |
| 2131 | # cpass0.dcsTree.list, cpass1.dcsTree.list |
| 2132 | # QA trending root file: |
| 2133 | # trending.root |
| 2134 | # |
| 2135 | # Production infoo ascii files: |
| 2136 | # summary_pass0.tree |
| 2137 | # summary_pass1.tree |
| 2138 | # |
| 617f8886 |
2139 | |
| 71353a5f |
2140 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 |
| 2141 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 |
| 2142 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 |
| 2143 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 |
| 2144 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 |
| 2145 | |
| 2146 | #first, dump the C macro to file |
| 2147 | cat << EOF > mergeTree.C |
| 2148 | // |
| 2149 | // Merge summary information |
| 2150 | // Following files are expected to be in the working directory |
| 2151 | // |
| 2152 | // Calibration file lists: |
| 2153 | // cpass0.dcsTree.list, cpass1.dcsTree.list |
| 2154 | // QA trending root file: |
| 2155 | // trending.root |
| 2156 | // |
| 2157 | // Production infoo ascii files: |
| 2158 | // summary_pass0.tree |
| 2159 | // summary_pass1.tree |
| 2160 | // |
| 2161 | void mergeTree(){ |
| 2162 | // |
| 2163 | // |
| 2164 | // |
| 2165 | // Calibration values dump |
| 2166 | // |
| 839b07ca |
2167 | //Printf("MakeTreeFromList cpass0.dcsTree.list"); |
| 71353a5f |
2168 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); |
| 839b07ca |
2169 | //Printf("MakeTreeFromList cpass1.dcsTree.list"); |
| 71353a5f |
2170 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); |
| 2171 | // |
| 2172 | // Calibration status dump |
| 2173 | // |
| 2174 | TFile *fprod = TFile::Open("fproduction.root","recreate"); |
| 2175 | TTree tree0, tree1; |
| 839b07ca |
2176 | //Printf("reading summary_pass0.tree"); |
| 71353a5f |
2177 | tree0.ReadFile("summary_pass0.tree"); |
| 839b07ca |
2178 | //Printf("reading summary_pass1.tree"); |
| 71353a5f |
2179 | tree1.ReadFile("summary_pass1.tree"); |
| 2180 | tree0.Write("CPass0"); |
| 2181 | tree1.Write("CPass1"); |
| 2182 | fprod->Close(); |
| 2183 | // |
| 2184 | // |
| 2185 | // |
| 2186 | TString stringSetup=""; |
| 2187 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // |
| 2188 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // |
| 2189 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // |
| 2190 | // |
| 2191 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // |
| 2192 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // |
| 2193 | // |
| 839b07ca |
2194 | //Printf("stringSetup: %s", stringSetup.Data()); |
| 71353a5f |
2195 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); |
| 2196 | } |
| 2197 | EOF |
| 2198 | |
| 2199 | aliroot -b -q "mergeTree.C" > mergeTrees.log |
| 2200 | return $? |
| 78d0bbfd |
2201 | ) |
| 71353a5f |
2202 | |
| 177b9ebc |
2203 | stackTraceTree() |
| 2204 | ( |
| e00eb304 |
2205 | if [[ $# -lt 1 ]]; then |
| 2206 | echo 'make stacktrace processing in case of standard root crash log' |
| 2207 | echo 'input is a (list of) text files with the stack trace (either gdb aoutput' |
| 2208 | echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,' |
| 2209 | echo 'or the root crash log), output is a TTree formatted table.' |
| 2210 | echo 'example usage:' |
| 2211 | echo 'benchmark.sh stackTraceTree /foo/*/rec.log' |
| 2212 | echo 'benchmark.sh stackTraceTree $(cat file.list)' |
| 0685f982 |
2213 | echo 'benchmark.sh stackTraceTree `cat file.list`' |
| e00eb304 |
2214 | return 0 |
| 2215 | fi |
| 4a3c2422 |
2216 | gawk ' |
| 177b9ebc |
2217 | BEGIN { |
| 80c75839 |
2218 | print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C"; |
| 177b9ebc |
2219 | RS="#[0-9]*"; |
| 2220 | aliroot=0; |
| 4a3c2422 |
2221 | read=1; |
| 177b9ebc |
2222 | } |
| 4a3c2422 |
2223 | /There was a crash/ {read=1;} |
| 2224 | /The lines below might hint at the cause of the crash/ {read=0;} |
| 2225 | read==1 { |
| 177b9ebc |
2226 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
| 2227 | gsub("#","",RT); |
| 80c75839 |
2228 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME |
| 177b9ebc |
2229 | } |
| 2230 | ' "$@" 2>/dev/null |
| 2231 | ) |
| 2232 | |
| 71353a5f |
2233 | goMakeSummary() |
| 78d0bbfd |
2234 | ( |
| 71353a5f |
2235 | #all the final stuff goes in here for ease of use: |
| 2236 | # summary logs |
| 2237 | # qa plot making |
| 2238 | # final file lists |
| ef77d296 |
2239 | # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries |
| 2240 | # will appear in the submission dir. |
| 444b9916 |
2241 | #some defaults: |
| 2242 | log="summary.log" |
| 2243 | productionID="qa" |
| 71353a5f |
2244 | |
| 2245 | configFile=${1} |
| 2246 | shift 1 |
| 444b9916 |
2247 | extraOpts=("$@") |
| 6fc48f9e |
2248 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
| 71353a5f |
2249 | |
| 0eaf38c7 |
2250 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi |
| 839b07ca |
2251 | |
| 39527a15 |
2252 | #record the working directory provided by the batch system |
| 2253 | batchWorkingDirectory=${PWD} |
| ef77d296 |
2254 | |
| 2255 | logTmp=${batchWorkingDirectory}/${log} |
| 2256 | logDest=${commonOutputPath}/${log} |
| 39527a15 |
2257 | |
| 71353a5f |
2258 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
| 2259 | |
| 71353a5f |
2260 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return |
| 2261 | |
| 2262 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
| 2263 | |
| 39527a15 |
2264 | #copy some useful stuff |
| 2265 | #and go to the commonOutputPath |
| 2266 | cp ${configFile} ${commonOutputPath} |
| 39527a15 |
2267 | |
| ef77d296 |
2268 | exec &> >(tee ${logTmp}) |
| 39527a15 |
2269 | |
| 71353a5f |
2270 | #summarize the global stuff |
| 2271 | echo "env script: ${alirootSource} ${alirootEnv}" |
| 2272 | echo "\$ALICE_ROOT=${ALICE_ROOT}" |
| 2273 | echo "commonOutputPath=${commonOutputPath}" |
| 2274 | |
| 2275 | #summarize the stacktraces |
| ef77d296 |
2276 | stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree |
| 2277 | stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree |
| 71353a5f |
2278 | |
| 2279 | echo total numbers for the production: |
| 2280 | echo |
| 2281 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 2282 | /^calibfile/ {nFiles++;} |
| 2283 | /core dumped/ {nCore++i;} |
| ef77d296 |
2284 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
| 71353a5f |
2285 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2286 | /\/rec.log OK/ {nOK++;} |
| 2287 | /\/rec.log BAD/ {nBAD++;} |
| 2288 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| ef77d296 |
2289 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
| 71353a5f |
2290 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2291 | /\/calib.log OK/ {nOK++;} |
| 2292 | /\/calib.log BAD/ {nBAD++;} |
| ef77d296 |
2293 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
| 71353a5f |
2294 | |
| 2295 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2296 | /merge.log OK/ {nOK++;} |
| 2297 | /merge.log BAD/ {nBAD++;} |
| ef77d296 |
2298 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null |
| 71353a5f |
2299 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2300 | /ocdb.log OK/ {nOK++;} |
| 2301 | /ocdb.log BAD/ {nBAD++;} |
| ef77d296 |
2302 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null |
| 71353a5f |
2303 | |
| 2304 | echo |
| 2305 | awk 'BEGIN {nFiles=0;nCore=0;} |
| 2306 | /^calibfile/ {nFiles++;} |
| 2307 | /core dumped/ {nCore++;} |
| ef77d296 |
2308 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
| 71353a5f |
2309 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2310 | /\/rec.log OK/ {nOK++;} |
| 2311 | /\/rec.log BAD/ {nBAD++;} |
| 2312 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} |
| ef77d296 |
2313 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
| 71353a5f |
2314 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2315 | /\/calib.log OK/ {nOK++;} |
| 2316 | /\/calib.log BAD/ {nBAD++;} |
| ef77d296 |
2317 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
| 71353a5f |
2318 | |
| 2319 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2320 | /merge.log OK/ {nOK++;} |
| 2321 | /merge.log BAD/ {nBAD++;} |
| ef77d296 |
2322 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null |
| 71353a5f |
2323 | awk 'BEGIN {nOK=0; nBAD=0; } |
| 2324 | /ocdb.log OK/ {nOK++;} |
| 2325 | /ocdb.log BAD/ {nBAD++;} |
| ef77d296 |
2326 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null |
| 71353a5f |
2327 | |
| 2328 | echo |
| 2329 | echo per run stats: |
| ef77d296 |
2330 | /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x |
| 71353a5f |
2331 | do |
| 5a2ee6ff |
2332 | dir=$(goPrintValues dir - ${x}) |
| 71353a5f |
2333 | runNumber=$(guessRunNumber ${dir}) |
| 2334 | [[ -z ${runNumber} ]] && continue |
| 2335 | |
| ef77d296 |
2336 | if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then |
| 71353a5f |
2337 | statusCPass0=( $( |
| 2338 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} |
| 2339 | /\/rec.log OK/ {nOKrec++;} |
| 2340 | /\/rec.log BAD/ {nBADrec++;} |
| 2341 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} |
| 2342 | /stderr OK/ {nOKstderr++;} |
| 2343 | /\/calib.log OK/ {nOKcalib++;} |
| 2344 | /\/calib.log BAD/ {nBADcalib++} |
| ef77d296 |
2345 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null |
| 71353a5f |
2346 | ) ) |
| 2347 | fi |
| 2348 | |
| ef77d296 |
2349 | if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then |
| 71353a5f |
2350 | statusCPass1=( $( |
| 2351 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} |
| 2352 | /\/rec.log OK/ {nOKrec++;} |
| 2353 | /\/rec.log BAD/ {nBADrec++;} |
| 2354 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} |
| 2355 | /stderr OK/ {nOKstderr++;} |
| 2356 | /\/calib.log OK/ {nOKcalib++;} |
| 2357 | /\/calib.log BAD/ {nBADcalib++} |
| 2358 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} |
| 2359 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} |
| 2360 | /\/qa_outer.log OK/ {nQAouterOK++;} |
| 2361 | /\/qa_outer.log BAD/ {nQAouterBAD++;} |
| ef77d296 |
2362 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null |
| 71353a5f |
2363 | ) ) |
| 2364 | fi |
| 2365 | |
| 2366 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) |
| 2367 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 2368 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) |
| 2369 | |
| 2370 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} |
| 2371 | done |
| 2372 | |
| 2373 | #make lists with output files - QA, trending, filtering and calibration |
| 2374 | ### wait for the merging of all runs to be over ### |
| 2375 | rm -f qa.list |
| 2376 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 2377 | rm -f calib.list |
| 2378 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 2379 | rm -f trending.list |
| 2380 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 2381 | rm -f filtering.list |
| 444b9916 |
2382 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 71353a5f |
2383 | rm -f cpass0.dcsTree.list |
| 2384 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null |
| 71353a5f |
2385 | rm -f cpass1.dcsTree.list |
| 2386 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
| 2387 | |
| 2388 | #merge trending |
| fecdbe75 |
2389 | rm -f trending.root |
| 2390 | goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log |
| 71353a5f |
2391 | |
| 2392 | goMakeSummaryTree ${commonOutputPath} 0 |
| 2393 | goMakeSummaryTree ${commonOutputPath} 1 |
| 2394 | |
| 839b07ca |
2395 | goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log |
| 71353a5f |
2396 | |
| 2397 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees |
| 2398 | goMakeMergedSummaryTree |
| 2399 | |
| 2400 | #if set, email the summary |
| ef77d296 |
2401 | [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO} |
| 2402 | |
| 2403 | # Copy log to destination (delete all on failure to signal error) |
| 2404 | cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest" |
| fecdbe75 |
2405 | |
| 2406 | #copy output files |
| 054fdec5 |
2407 | exec &> >(tee fileCopy.log) |
| 13abb339 |
2408 | paranoidCp QAplots ${commonOutputPath} |
| 2409 | paranoidCp *.list ${commonOutputPath} |
| 2410 | paranoidCp *.root ${commonOutputPath} |
| 2411 | paranoidCp *.log ${commonOutputPath} |
| 054fdec5 |
2412 | paranoidCp fileCopy.log ${commonOutputPath} |
| 71353a5f |
2413 | |
| 2414 | return 0 |
| 78d0bbfd |
2415 | ) |
| 71353a5f |
2416 | |
| 2417 | goMakeSummaryTree() |
| 78d0bbfd |
2418 | ( |
| 71353a5f |
2419 | if [[ $# -lt 1 ]] ; then |
| 2420 | return |
| 2421 | fi |
| 2422 | #1. define vars/arrays |
| 2423 | DIR=${1} #use input or exec in current dir |
| 2424 | pass=${2-"0"} #pass from input |
| 2425 | outfile="summary_pass${pass}.tree" |
| 2426 | Ncolumns=0 |
| 2427 | test -f ${outfile} && : >${outfile} |
| 2428 | errfile=${outfile/tree/err} |
| 2429 | test -f ${errfile} && : >${errfile} |
| 2430 | |
| 2431 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) |
| 2432 | Ncounter=${#counterString[@]} |
| 2433 | |
| 2434 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) |
| 2435 | Nstatus=${#statusString[@]} |
| 2436 | |
| 2437 | |
| 2438 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) |
| 2439 | Nrates=${#ratesString[@]} |
| 2440 | |
| 2441 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) |
| 2442 | Nruns=${#runs[@]} |
| 2443 | |
| 2444 | echo -n runnumber/I >>${outfile} |
| 2445 | echo -n :cpass${pass}status/I >>${outfile} |
| 2446 | echo -n :cpass${pass}QAstatus/I >>${outfile} |
| 2447 | for i in ${ratesString[@]}; do |
| 2448 | echo -n :${i}OK/I >>${outfile} |
| 2449 | echo -n :${i}BAD/I >>${outfile} |
| 2450 | |
| 2451 | done |
| 2452 | for i in ${counterString[@]} ${statusString[@]} ; do |
| 2453 | echo -n :${i}/I >>${outfile} |
| 2454 | done |
| 2455 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) |
| 2456 | echo >> ${outfile} |
| 2457 | |
| 2458 | #2. loop runs |
| 2459 | |
| 2460 | for runnumber in ${runs[@]} ; do |
| 2461 | |
| 2462 | |
| 2463 | |
| 2464 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" |
| 2465 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" |
| 2466 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') |
| 2467 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then |
| 2468 | echo "${filemerge} does not exist!" >>${errfile} |
| 2469 | continue |
| 2470 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then |
| 2471 | echo "${filejob} does not exist!" >>${errfile} |
| 2472 | echo -n ${runnumber} >> ${outfile} |
| 2473 | for i in $(seq ${Ncolumns}) ; do |
| 2474 | echo -n "-1" >> ${outfile} |
| 2475 | done |
| 2476 | echo >> ${outfile} |
| 2477 | continue |
| 2478 | fi |
| 2479 | echo -n ${runnumber} >> ${outfile} |
| 2480 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' |
| 2481 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) |
| 2482 | echo -n " ${passStatus}" >> ${outfile} |
| 2483 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) |
| 2484 | echo -n " ${qaStatus}" >> ${outfile} |
| 2485 | |
| 2486 | |
| 2487 | #fill OK/BAD rates |
| 2488 | for i in $(seq 0 $((${Nrates}-1))) ; do |
| 2489 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) |
| 2490 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) |
| 2491 | |
| 2492 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then |
| 2493 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) |
| 2494 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) |
| 2495 | fi |
| 2496 | echo -n " ${var1}" >> ${outfile} |
| 2497 | echo -n " ${var2}" >> ${outfile} |
| 2498 | done |
| 2499 | |
| 2500 | if [[ -f ${fileOCDB} ]] ; then |
| 2501 | #fill counter |
| 2502 | for i in $(seq 0 $((${Ncounter}-1))) ; do |
| 2503 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) |
| 2504 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2505 | done |
| 2506 | |
| 2507 | #fill status |
| 2508 | for i in $(seq 0 $((${Nstatus}-1))) ; do |
| 2509 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) |
| 2510 | echo -n " ${var1:-"-1"}" >> ${outfile} |
| 2511 | done |
| 2512 | fi |
| 2513 | echo >> ${outfile} |
| 2514 | done |
| 2515 | |
| 5de3064b |
2516 | return 0 |
| 78d0bbfd |
2517 | ) |
| 71353a5f |
2518 | |
| 2519 | parseConfig() |
| 2520 | { |
| 2521 | configFile=${1} |
| 2522 | shift |
| 444b9916 |
2523 | args=("$@") |
| 2524 | |
| 0eaf38c7 |
2525 | |
| 444b9916 |
2526 | #some defaults |
| 6fc48f9e |
2527 | #autoOCDB=0 |
| 2528 | defaultOCDB="raw://" |
| 2529 | #runNumber=167123 |
| 2530 | #makeflowPath="/hera/alice/aux/cctools/bin" |
| 2531 | #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097" |
| 2532 | #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097" |
| 2533 | makeflowOptions="" |
| 39527a15 |
2534 | #batchCommand="/usr/bin/qsub" |
| ef77d296 |
2535 | batchFlags="" |
| 6fc48f9e |
2536 | baseOutputDirectory="$PWD/output" |
| 2537 | #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c" |
| 2538 | #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh" |
| 2539 | #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C' |
| 2540 | reconstructInTemporaryDir=0 |
| 2541 | recoTriggerOptions="\"\"" |
| 2542 | percentProcessedFilesToContinue=100 |
| 2543 | maxSecondsToWait=$(( 3600*24 )) |
| 2544 | nEvents=-1 |
| 2545 | nMaxChunks=0 |
| 2546 | postSetUpActionCPass0="" |
| 2547 | postSetUpActionCPass1="" |
| 2548 | runCPass0reco=1 |
| 2549 | runCPass0MergeMakeOCDB=1 |
| 2550 | runCPass1reco=1 |
| 2551 | runCPass1MergeMakeOCDB=1 |
| 2552 | runESDfiltering=1 |
| 444b9916 |
2553 | filteringFactorHighPt=1e2 |
| 2554 | filteringFactorV0s=1e1 |
| 6fc48f9e |
2555 | MAILTO="" |
| 2556 | #pretend=1 |
| 2557 | #dontRedirectStdOutToLog=1 |
| 2558 | logToFinalDestination=1 |
| 2559 | ALIROOT_FORCE_COREDUMP=1 |
| 0eaf38c7 |
2560 | pretendDelay=0 |
| 71353a5f |
2561 | |
| 2562 | #first, source the config file |
| 2563 | if [ -f ${configFile} ]; then |
| 2564 | source ${configFile} |
| 2565 | else |
| 0eaf38c7 |
2566 | echo "config file ${configFile} not found!" |
| 2567 | return 1 |
| 71353a5f |
2568 | fi |
| 2569 | |
| 839b07ca |
2570 | unset encodedSpaces |
| 2571 | for opt in "${args[@]}"; do |
| 0eaf38c7 |
2572 | [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break |
| 839b07ca |
2573 | done |
| 2574 | |
| d56eeaab |
2575 | #then, parse the options as they override the options from file |
| 2576 | for opt in "${args[@]}"; do |
| 0eaf38c7 |
2577 | [[ -z ${opt} ]] && continue |
| 2578 | [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})" |
| 2579 | [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue |
| d56eeaab |
2580 | if [[ ! "${opt}" =~ .*=.* ]]; then |
| 0eaf38c7 |
2581 | echo "badly formatted option \"${opt}\" should be: option=value, stopping..." |
| d56eeaab |
2582 | return 1 |
| 2583 | fi |
| 2584 | local var="${opt%%=*}" |
| 2585 | local value="${opt#*=}" |
| 9d0c204f |
2586 | echo "${var}=${value}" |
| 444b9916 |
2587 | export ${var}="${value}" |
| 71353a5f |
2588 | done |
| 2589 | |
| 6fc48f9e |
2590 | #do some checking |
| 2591 | [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1 |
| 2592 | |
| 71353a5f |
2593 | #export the aliroot function if defined to override normal behaviour |
| fecdbe75 |
2594 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..." |
| 6fc48f9e |
2595 | |
| 5de3064b |
2596 | return 0 |
| 71353a5f |
2597 | } |
| 2598 | |
| 2599 | aliroot() |
| 2600 | { |
| e48b4924 |
2601 | args=("$@") |
| 6fc48f9e |
2602 | if [[ -n ${useProfilingCommand} ]]; then |
| e48b4924 |
2603 | profilerLogFile="cpu.txt" |
| 1a0fcd62 |
2604 | [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt" |
| 2605 | [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt" |
| 2f2d99b8 |
2606 | echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}" |
| e48b4924 |
2607 | ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile} |
| 71353a5f |
2608 | else |
| 2609 | #to prevent an infinite recursion use "command aliroot" to disable |
| 2610 | #aliases and functions |
| e48b4924 |
2611 | echo running command aliroot "${args[@]}" |
| 2612 | command aliroot "${args[@]}" |
| 71353a5f |
2613 | fi |
| 5de3064b |
2614 | return 0 |
| 71353a5f |
2615 | } |
| 2616 | |
| 13abb339 |
2617 | paranoidCp() |
| 2618 | ( |
| 2619 | #recursively copy files and directories |
| 2620 | #to avoid using find and the like as they kill |
| 2621 | #the performance on some cluster file systems |
| 2622 | #does not copy links to avoid problems |
| 2623 | sourceFiles=("${@}") |
| 2624 | destination="${sourceFiles[@]:(-1)}" #last element |
| 2625 | unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst) |
| 2626 | for src in "${sourceFiles[@]}"; do |
| 2627 | if [[ -f "${src}" && ! -h "${src}" ]]; then |
| 2628 | paranoidCopyFile "${src}" "${destination}" |
| 2629 | elif [[ -d "${src}" && ! -h "${src}" ]]; then |
| 2630 | src="${src%/}" |
| 2631 | dst="${destination}/${src##*/}" |
| 2632 | mkdir -p "${dst}" |
| 2633 | paranoidCp "${src}"/* "${dst}" |
| 2634 | fi |
| 2635 | done |
| 2636 | ) |
| 2637 | |
| 2638 | paranoidCopyFile() |
| 2639 | ( |
| 2640 | #copy a single file to a target in an existing dir |
| 2641 | #repeat a few times if copy fails |
| 2642 | src="${1}" |
| 2643 | dst="${2}" |
| 2644 | [[ -d "${dst}" ]] && dst="${dst}/${src##*/}" |
| 2645 | [[ -z "${maxCopyTries}" ]] && maxCopyTries=5 |
| 054fdec5 |
2646 | #echo "maxCopyTries=${maxCopyTries}" |
| 13abb339 |
2647 | echo "cp ${src} ${dst}" |
| 2648 | cp "${src}" "${dst}" |
| 2649 | i=0 |
| 2650 | until cmp -s "${src}" "${dst}"; do |
| 2651 | echo "try: ${i}" |
| 2652 | [[ -f "${dst}" ]] && rm "${dst}" |
| 2653 | cp "${src}" "${dst}" |
| 2654 | [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1 |
| 2655 | (( i++ )) |
| 2656 | done |
| 2657 | return 0 |
| 2658 | ) |
| 2659 | |
| 71353a5f |
2660 | guessRunData() |
| 2661 | { |
| 2662 | #guess the period from the path, pick the rightmost one |
| 2663 | period="" |
| 2664 | runNumber="" |
| 2665 | year="" |
| 2666 | pass="" |
| 2667 | legoTrainRunNumber="" |
| 2668 | dataType="" |
| 2669 | |
| 2670 | local shortRunNumber="" |
| 2671 | local IFS="/" |
| 2672 | declare -a path=( $1 ) |
| 2673 | local dirDepth=$(( ${#path[*]}-1 )) |
| 2674 | i=0 |
| 2675 | #for ((x=${dirDepth};x>=0;x--)); do |
| 2676 | for ((x=0;x<=${dirDepth};x++)); do |
| 2677 | |
| 2678 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} |
| 2679 | local field=${path[${x}]} |
| 2680 | local fieldNext=${path[$((x+1))]} |
| 2681 | |
| 2682 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} |
| 2683 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} |
| 2684 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} |
| 2685 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} |
| 2686 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} |
| 2687 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} |
| 2688 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} |
| 2689 | (( i++ )) |
| 2690 | done |
| 2691 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} |
| 2692 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} |
| 2693 | |
| 2694 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; |
| 2695 | if [[ -z ${runNumber}} ]]; |
| 2696 | then |
| 2697 | #error condition |
| 2698 | return 1 |
| 2699 | else |
| 2700 | #ALL OK |
| 2701 | return 0 |
| 2702 | fi |
| 5de3064b |
2703 | return 0 |
| 71353a5f |
2704 | } |
| 2705 | |
| 839b07ca |
2706 | #these functions encode strings to and from a space-less form |
| 2707 | #use when spaces are not well handled (e.g. in arguments to |
| 2708 | #commands in makeflow files, etc. |
| 0eaf38c7 |
2709 | encSpaces()(echo "${1// /@@@@}") |
| 2710 | decSpaces()(echo "${1//@@@@/ }") |
| 839b07ca |
2711 | |
| fc7e1e45 |
2712 | main "$@" |
git://git.uio.no / u / mrichter / AliRoot.git / blame