when will it end??
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
fc7e1e45 4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
71353a5f 10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
fc7e1e45 24 umask 0002
25 shift
71353a5f 26 case ${runMode} in
fc7e1e45 27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
71353a5f 29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 33 "MakeSummary") goMakeSummary "$@";;
cb473870 34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
fc7e1e45 36 "test") goTest "$@";;
71353a5f 37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
cb473870 39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 41 "Merge") goMerge "$@";;
42 *)
444b9916 43 ${runMode} "$@"
fc7e1e45 44 ;;
45 esac
39527a15 46 return 0
fc7e1e45 47}
48
71353a5f 49generateMC()
50{
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
54 runNumber=${1}
55 OCDBpath=${2}
56 nEventsim=${3}
57 if [[ -n ${pretend} ]]; then
0eaf38c7 58 sleep ${pretendDelay}
71353a5f 59 touch galice.root
60 else
61 if [[ -f sim.C && -f Config.C ]] ; then
62 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63 mv syswatch.log simwatch.log
64 fi
65 fi
66}
67
fc7e1e45 68goCPass0()
78d0bbfd 69(
fc7e1e45 70 umask 0002
71
71353a5f 72 targetDirectory=${1}
73 inputList=${2}
74 nEvents=${3}
75 ocdbPath=${4}
76 configFile=${5}
77 runNumber=${6}
78 jobindex=${7}
79 shift 7
6fc48f9e 80 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 81
39527a15 82 #record the working directory provided by the batch system
83 batchWorkingDirectory=${PWD}
84
71353a5f 85 #use the jobindex only if set and non-negative
86 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89 fi
90
91 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
92 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
93
94 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
95
96 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
97 ulimit -c unlimited
98 export ALIROOT_FORCE_COREDUMP
99 fi
fc7e1e45 100
cb473870 101 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
102 #at the QA stage the pass number is guessed from the path stored here.
103 #The Format is:
104 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
105 export PRODUCTION_METADATA="OutputDir=cpass0"
106
5de3064b 107 if [[ "${inputList}" =~ \.root$ ]]; then
108 infile=${inputList}
109 else
110 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
111 fi
71353a5f 112 chunkName=${infile##*/}
71353a5f 113
114 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
115 mkdir -p ${outputDir}
116 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
117
118 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
119 runpath=${outputDir}
0eaf38c7 120 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
121 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
71353a5f 122 mkdir -p ${runpath}
123 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
0eaf38c7 124 if ! cd ${runpath}; then
125 echo "PWD=$PWD is not the runpath=${runpath}"
126 touch ${doneFile}
127 return 1
128 fi
fc7e1e45 129
5de3064b 130 #runCPassX/C expects the raw chunk to be linked in the run dir
131 #despite it being accessed by the full path
132 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 133
71353a5f 134 #####MC
135 if [[ -n ${generateMC} ]]; then
136 olddir=${PWD}
137 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
138 simrunpath=${outputDirMC}
0eaf38c7 139 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
140 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
71353a5f 141 mkdir -p ${outputDirMC}
142 mkdir -p ${simrunpath}
0eaf38c7 143 if cd ${simrunpath}; then
fc7e1e45 144
0eaf38c7 145 filesMC=(
146 "${batchWorkingDirectory}/sim.C"
147 "${batchWorkingDirectory}/rec.C"
148 "${batchWorkingDirectory}/Config.C"
149 "${batchWorkingDirectory}/OCDB_*.root"
150 )
151 for file in ${filesMC[*]}; do
152 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
153 done
fc7e1e45 154
0eaf38c7 155 generateMC ${runNumber} ${ocdbPath} ${nEvents}
156
157 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
158 cd ${olddir}
159
160 ln -s ${outputDirMC}/* ${runpath}/
161
162 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
163 infile=""
164 fi
71353a5f 165 fi
166 ######
167
168 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
169
170 logOutputDir=${runpath}
171 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 172 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
173 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 174 echo "${0} $*"
fc7e1e45 175
176 echo "#####################"
177 echo CPass0:
178 echo JOB setup
71353a5f 179 echo nEvents ${nEvents}
180 echo runNumber ${runNumber}
181 echo ocdbPath ${ocdbPath}
182 echo infile ${infile}
183 echo chunkName ${chunkName}
184 echo jobindex ${jobindex}
185 echo recoTriggerOptions ${recoTriggerOptions}
186 echo targetDirectory ${targetDirectory}
187 echo commonOutputPath ${commonOutputPath}
188 echo doneFile ${doneFile}
189 echo runpath ${runpath}
190 echo outputDir ${outputDir}
191 echo ALICE_ROOT ${ALICE_ROOT}
192 echo PWD ${PWD}
fc7e1e45 193 echo "########## ###########"
194
71353a5f 195 alirootInfo > ALICE_ROOT.log
fc7e1e45 196
197 filesCPass0=(
39527a15 198 "${batchWorkingDirectory}/runCPass0.sh"
199 "${batchWorkingDirectory}/recCPass0.C"
200 "${batchWorkingDirectory}/runCalibTrain.C"
201 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
202 "${batchWorkingDirectory}/OCDB.root"
71353a5f 203 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
204 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
205 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 206 )
207
208 for file in ${filesCPass0[*]}; do
209 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 210 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 211 done
212
71353a5f 213 echo "this directory (${PWD}) contents:"
214 /bin/ls
fc7e1e45 215 echo
216 chmod u+x runCPass0.sh
217
71353a5f 218 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
219
220 if [[ -n ${postSetUpActionCPass0} ]]; then
221 echo "running ${postSetUpActionCPass0}"
222 eval ${postSetUpActionCPass0}
fc7e1e45 223 fi
224
225 #run CPass0
71353a5f 226 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
227 if [[ -n ${pretend} ]]; then
0eaf38c7 228 sleep ${pretendDelay}
cb473870 229 touch AliESDs.root
230 touch AliESDfriends.root
fc7e1e45 231 touch AliESDfriends_v1.root
232 touch rec.log
233 touch calib.log
234 else
71353a5f 235 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
236 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 237 fi
238
fc7e1e45 239 #move stuff to final destination
71353a5f 240 echo "this directory (${PWD}) contents:"
241 /bin/ls
fc7e1e45 242 echo
243
71353a5f 244 echo rm -f ./${chunkName}
245 rm -f ./${chunkName}
0eaf38c7 246 echo "cp -R ${runpath}/* ${outputDir}"
247 cp -p -R ${runpath}/* ${outputDir}
fc7e1e45 248 echo
5ed7c82c 249
250 #validate CPass0
251 cd ${outputDir}
71353a5f 252 touch ${doneFile}
71353a5f 253 echo "dir ${outputDir}" >> ${doneFile}
ab946185 254 if summarizeLogs >> ${doneFile}; then
255 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
256 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
257 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
258 fi
fc7e1e45 259
0eaf38c7 260 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
5de3064b 261 return 0
78d0bbfd 262)
fc7e1e45 263
264goCPass1()
78d0bbfd 265(
fc7e1e45 266 umask 0002
267
71353a5f 268 targetDirectory=${1}
269 inputList=${2}
270 nEvents=${3}
271 ocdbPath=${4}
272 configFile=${5}
273 runNumber=${6}
274 jobindex=${7}
275 shift 7
444b9916 276 extraOpts=("$@")
6fc48f9e 277 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 278
39527a15 279 #record the working directory provided by the batch system
280 batchWorkingDirectory=${PWD}
281
71353a5f 282 #use the jobindex only if set and non-negative
283 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
284 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
285 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
286 fi
287
288 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
289 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
290
291 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
292
293 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
294 ulimit -c unlimited
295 export ALIROOT_FORCE_COREDUMP
296 fi
fc7e1e45 297
cb473870 298 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
299 #at the QA stage the pass number is guessed from the path stored here.
300 #The Format is:
301 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
302 export PRODUCTION_METADATA="OutputDir=cpass1"
303
71353a5f 304 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
fc7e1e45 305 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 306 infile=${inputList}
fc7e1e45 307 else
71353a5f 308 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 309 fi
fc7e1e45 310 chunkName=${infile##*/}
5de3064b 311
71353a5f 312 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
313 mkdir -p ${outputDir}
314 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
fc7e1e45 315
71353a5f 316 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
317 runpath=${outputDir}
0eaf38c7 318 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
319 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
71353a5f 320
321 #MC
322 if [[ "${infile}" =~ galice\.root ]]; then
323 ln -s ${inputList%/*}/* ${runpath}
324 infile=""
325 fi
fc7e1e45 326
327 #init the running path
71353a5f 328 mkdir -p ${runpath}
329 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
0eaf38c7 330 if ! cd ${runpath}; then
331 echo "PWD=$PWD is not the runpath=${runpath}"
332 touch ${doneFile}
333 return 1
334 fi
fc7e1e45 335
71353a5f 336 #this is needed for runCPass1.sh
337 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 338
71353a5f 339 logOutputDir=${runpath}
340 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 341 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
342 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 343 echo "${0} $*"
fc7e1e45 344
345 echo "#####################"
346 echo CPass1:
347 echo JOB setup
71353a5f 348 echo nEvents ${nEvents}
349 echo runNumber ${runNumber}
350 echo ocdbPath ${ocdbPath}
351 echo infile ${infile}
352 echo chunkName ${chunkName}
353 echo jobindex ${jobindex}
354 echo recoTriggerOptions ${recoTriggerOptions}
355 echo targetDirectory ${targetDirectory}
356 echo commonOutputPath ${commonOutputPath}
357 echo doneFile ${doneFile}
358 echo runpath ${runpath}
359 echo outputDir ${outputDir}
78d0bbfd 360 echo batchWorkingDirectory ${batchWorkingDirectory}
71353a5f 361 echo ALICE_ROOT ${ALICE_ROOT}
362 echo PWD ${PWD}
fc7e1e45 363 echo "########## ###########"
364
71353a5f 365 alirootInfo > ALICE_ROOT.log
fc7e1e45 366
367 filesCPass1=(
39527a15 368 "${batchWorkingDirectory}/runCPass1.sh"
369 "${batchWorkingDirectory}/recCPass1.C"
370 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
371 "${batchWorkingDirectory}/runCalibTrain.C"
372 "${batchWorkingDirectory}/QAtrain_duo.C"
373 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
0eaf38c7 374 "${batchWorkingDirectory}/${configFile}"
617f8886 375 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 376 "${batchWorkingDirectory}/OCDB.root"
71353a5f 377 "${trustedQAtrainMacro}"
378 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
379 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
380 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
381 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
382 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 383 )
384
78d0bbfd 385 for file in "${filesCPass1[@]}"; do
fc7e1e45 386 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
78d0bbfd 387 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
fc7e1e45 388 done
389
71353a5f 390 echo "this directory (${PWD}) contents:"
391 /bin/ls
fc7e1e45 392 echo
393
78d0bbfd 394 #remove spaces around commas from calls to root
71353a5f 395 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
396
397 if [[ -n ${postSetUpActionCPass1} ]]; then
398 echo "running ${postSetUpActionCPass1}"
399 eval ${postSetUpActionCPass1}
fc7e1e45 400 echo
401 fi
402
403 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
404 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 405 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
406 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 407 else
408 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
409 fi
410
71353a5f 411 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
cb473870 412 echo "cpass0 produced no calibration! exiting..."
71353a5f 413 touch ${doneFile}
414 return 1
cb473870 415 fi
fc7e1e45 416
71353a5f 417 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
418 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
419 #entries using a relative path, e.g. local://./OCDB
78d0bbfd 420 echo "linking the OCDB/ for Barrel and OuterDet directories"
71353a5f 421 mkdir Barrel OuterDet
78d0bbfd 422 ls -l
71353a5f 423 ln -s ../OCDB Barrel/OCDB
424 ln -s ../OCDB OuterDet/OCDB
425
444b9916 426 #setup the filtering
427 #the following option enables the filtering task inside the QAtrain_duo.C
428 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
429 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
430 if [[ -n ${filteringFactorHighPt} ]]; then
431 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
432 fi
433 if [[ -n ${filteringFactorV0s} ]]; then
434 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
435 fi
436
fc7e1e45 437 #run CPass1
438 chmod u+x runCPass1.sh
71353a5f 439 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
440 if [[ -n ${pretend} ]]; then
0eaf38c7 441 sleep ${pretendDelay}
cb473870 442 touch AliESDs_Barrel.root
443 touch AliESDfriends_Barrel.root
fc7e1e45 444 touch AliESDfriends_v1.root
cb473870 445 touch QAresults_barrel.root
446 touch EventStat_temp_barrel.root
447 touch AODtpITS.root
448 touch AliESDs_Outer.root
449 touch AliESDfriends_Outer.root
450 touch QAresults_outer.root
451 touch EventStat_temp_outer.root
fc7e1e45 452 touch rec.log
453 touch calib.log
454 touch qa.log
444b9916 455 touch filtering.log FilterEvents_Trees.root
fc7e1e45 456 else
71353a5f 457 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 458
459 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
460 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
461 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
462 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
463 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
464 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
465 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
466 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
467
468 #make the filtered tree (if requested and not already produced by QA
469 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
fab31805 470 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
471 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
472 else
473 echo ""
444b9916 474 fi
475
fc7e1e45 476 fi
cb473870 477
478 ##handle possible crashes in QA (happens often in trunk)
479 ##rerun QA with a trusted aliroot version
480 #if [[ $(validateLog qa_barrel.log) ]]; then
481 # echo "qa_barrel.log not validated!"
482 #fi
483 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 484 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 485 # source ${setupTrustedAliROOTenvInCurrentShell}
486 # cd Barrel
487 # rm QAresults_barrel.root
488 # rm EventStat_temp_barrel.root
489 # rm AODtpITS.root
490 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
491 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 492 # if [[ -n ${trustedQAtrainMacro} ]]; then
493 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 494 # fi
71353a5f 495 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
496 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 497 # cd ../
498 #fi
cb473870 499
fc7e1e45 500 #move stuff to final destination
71353a5f 501 echo "this directory (${PWD}) contents:"
502 /bin/ls
503 echo rm -f ./${chunkName}
504 rm -f ./${chunkName}
0eaf38c7 505 echo "cp -R ${runpath}/* ${outputDir}"
506 cp -pf -R ${runpath}/* ${outputDir}
fc7e1e45 507 echo
508
5ed7c82c 509 #validate CPass1
510 cd ${outputDir}
71353a5f 511 touch ${doneFile}
71353a5f 512 echo "dir ${outputDir}" >> ${doneFile}
ab946185 513 if summarizeLogs >> ${doneFile}; then
514 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
515 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
516 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
517 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
34ef8842 518 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
519 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
ab946185 520 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
922d0b10 521 else
522 if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
523 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
524 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
525 fi
526 if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
527 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
528 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
529 fi
530 if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
531 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
532 fi
ab946185 533 fi
534
71353a5f 535 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 536 return 0
78d0bbfd 537)
fc7e1e45 538
539
540goMergeCPass0()
78d0bbfd 541(
fc7e1e45 542 #
543 # find the output files and merge them
544 #
545
71353a5f 546 outputDir=${1}
547 ocdbStorage=${2}
548 configFile=${3}
549 runNumber=${4}
550 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
551 shift 5
6fc48f9e 552 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 553
39527a15 554 #record the working directory provided by the batch system
555 batchWorkingDirectory=${PWD}
556
71353a5f 557 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
558 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
559
560 umask 0002
561 ulimit -c unlimited
fc7e1e45 562
71353a5f 563 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 564
71353a5f 565 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
566 runpath=${outputDir}
0eaf38c7 567 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
568 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
fc7e1e45 569
71353a5f 570 mkdir -p ${runpath}
571 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
0eaf38c7 572 if ! cd ${runpath}; then
573 echo "PWD=$PWD is not the runpath=${runpath}"
574 touch ${doneFile}
575 return 1
576 fi
fc7e1e45 577
71353a5f 578 logOutputDir=${runpath}
579 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
580 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
581 echo "${0} $*"
fc7e1e45 582
fc7e1e45 583 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 584
585 echo goMergeCPass0 SETUP:
71353a5f 586 echo runNumber=${runNumber}
587 echo outputDir=${outputDir}
588 echo ocdbStorage=${ocdbStorage}
589 echo calibrationFilesToMerge=${calibrationFilesToMerge}
590 echo mergingScript=${mergingScript}
591 echo commonOutputPath=${commonOutputPath}
592 echo runpath=${runpath}
fc7e1e45 593
594 # copy files in case they are not already there
595 filesMergeCPass0=(
39527a15 596 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
597 "${batchWorkingDirectory}/OCDB.root"
598 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 599 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
600 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
601 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
602 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
603 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 604 )
605 for file in ${filesMergeCPass0[*]}; do
606 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 607 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 608 done
609
71353a5f 610 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
611
612 alirootInfo > ALICE_ROOT.log
fc7e1e45 613
614 #
cb473870 615 echo "PWD"
71353a5f 616 /bin/ls
cb473870 617 echo "PWD/.."
71353a5f 618 /bin/ls ../
cb473870 619
fc7e1e45 620
621 #merge calibration
71353a5f 622 chmod u+x ${mergingScript}
fc7e1e45 623 mkdir -p ./OCDB
71353a5f 624 if [[ ! -f ${calibrationFilesToMerge} ]]; then
625 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
626 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 627 fi
628
71353a5f 629 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
630 if [[ -n ${pretend} ]]; then
0eaf38c7 631 sleep ${pretendDelay}
fc7e1e45 632 touch CalibObjects.root
633 touch ocdb.log
634 touch merge.log
617f8886 635 touch dcsTime.root
71353a5f 636 mkdir -p ./OCDB/TPC/Calib/TimeGain/
637 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
638 touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
639 touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 640 else
71353a5f 641 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
fc7e1e45 642
617f8886 643 #produce the calib trees for expert QA (dcsTime.root)
644 goMakeLocalOCDBaccessConfig ./OCDB
645 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
646 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
647 fi
648
fc7e1e45 649 ### produce the output
650 #tar the produced OCDB for reuse
71353a5f 651 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
fc7e1e45 652
71353a5f 653 /bin/ls
5ed7c82c 654
655 #copy all to output dir
0eaf38c7 656 cp -pf -R ${runpath}/* ${outputDir}
617f8886 657
71353a5f 658 if [[ -n ${generateMC} ]]; then
659 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
660 fi
661
fc7e1e45 662 #validate merging cpass0
5ed7c82c 663 cd ${outputDir}
71353a5f 664 touch ${doneFile}
71353a5f 665 echo "dir ${outputDir}" >> ${doneFile}
ab946185 666 if summarizeLogs >> ${doneFile}; then
667 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
668 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
669 fi
fc7e1e45 670
71353a5f 671 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 672 return 0
78d0bbfd 673)
fc7e1e45 674
675goMergeCPass1()
78d0bbfd 676(
fc7e1e45 677 #
678 # find the output files and merge them
679 #
680
71353a5f 681 outputDir=${1}
682 ocdbStorage=${2}
683 configFile=${3}
684 runNumber=${4}
685 calibrationFilesToMerge=${5}
686 qaFilesToMerge=${6}
444b9916 687 filteredFilesToMerge=${7}
688 shift 7
6fc48f9e 689 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 690
39527a15 691 #record the working directory provided by the batch system
692 batchWorkingDirectory=${PWD}
693
71353a5f 694 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
695 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
696
697 umask 0002
698 ulimit -c unlimited
fc7e1e45 699
700 #clean up first:
71353a5f 701 rm -f ${outputDir}/*.log
702 rm -f ${outputDir}/*.root
703 rm -f ${outputDir}/*done
704
705 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
706
707 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
708 runpath=${outputDir}
0eaf38c7 709 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
710 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
71353a5f 711
712 mkdir -p ${runpath}
713 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
0eaf38c7 714 if ! cd ${runpath}; then
715 echo "PWD=$PWD is not the runpath=${runpath}"
716 touch ${doneFile}
717 return 1
718 fi
71353a5f 719
720 logOutputDir=${runpath}
721 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
722 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
723 echo "${0} $*"
724
725 calibrationOutputFileName='AliESDfriends_v1.root'
726 qaOutputFileName='QAresults*.root'
fc7e1e45 727 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 728 #important to have the string "Stage.txt" in the filename to trigger the merging
729 #it has to be a list of directories containing the files
fc7e1e45 730 qaMergedOutputFileName="QAresults_merged.root"
731
732 echo goMergeCPass1 SETUP:
71353a5f 733 echo runNumber=${runNumber}
734 echo outputDir=${outputDir}
735 echo ocdbStorage=${ocdbStorage}
736 echo calibrationFilesToMerge=$calibrationFilesToMerge
737 echo qaFilesToMerge=$qaFilesToMerge
738 echo calibrationOutputFileName=${calibrationOutputFileName}
739 echo mergingScript=${mergingScript}
fc7e1e45 740
741 # copy files in case they are not already there
742 filesMergeCPass1=(
39527a15 743 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
744 "${batchWorkingDirectory}/${qaFilesToMerge}"
745 "${batchWorkingDirectory}/OCDB.root"
746 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
3fbef3b9 747 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 748 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 749 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
750 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
751 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
752 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
753 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
754 "${trustedQAtrainMacro}"
755 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 756 )
757 for file in ${filesMergeCPass1[*]}; do
758 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 759 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 760 done
761
71353a5f 762 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
763
cb473870 764 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
765 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 766 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
767 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 768 else
769 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
770 fi
771
71353a5f 772 alirootInfo > ALICE_ROOT.log
fc7e1e45 773
774 #
71353a5f 775 /bin/ls
fc7e1e45 776
777 #merge calibration
71353a5f 778 chmod u+x ${mergingScript}
fc7e1e45 779 mkdir -p OCDB
444b9916 780
781 #if not provided, create the lists of files to merge
782 if [[ ! -f ${filteredFilesToMerge} ]]; then
783 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
784 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
785 fi
71353a5f 786 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 787 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
788 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
789 fi
790 if [[ ! -f ${qaFilesToMerge} ]]; then
791 #find the files, but only store the directories (QAtrain_duo.C requires this)
792 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
793 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 794 fi
795
71353a5f 796 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
797 if [[ -n ${pretend} ]]; then
0eaf38c7 798 sleep ${pretendDelay}
fc7e1e45 799 touch ocdb.log
71353a5f 800 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 801 touch ${qaMergedOutputFileName}
fc7e1e45 802 touch merge.log
cb473870 803 touch trending.root
444b9916 804 touch FilterEvents_Trees.root
ab946185 805 touch CalibObjects.root
806 touch dcsTime.root
807 touch ${qaMergedOutputFileName}
617f8886 808 mkdir -p OCDB
fc7e1e45 809 else
617f8886 810 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
811
812 #merge QA (and filtered trees)
813 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
814 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
815
816 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
817 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 818 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
819 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
820 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 821 mv trending_barrel.root trending.root
444b9916 822
823 #merge filtered trees
824 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922d0b10 825 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
617f8886 826
827 #produce the calib trees for expert QA
828 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
922d0b10 829 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
fc7e1e45 830 fi
617f8886 831
832 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
71353a5f 833
834 /bin/ls
5ed7c82c 835
836 #copy all to output dir
0eaf38c7 837 cp -pf -R ${runpath}/* ${outputDir}
5ed7c82c 838
fc7e1e45 839 #validate merge cpass1
5ed7c82c 840 cd ${outputDir}
71353a5f 841 touch ${doneFile}
71353a5f 842 echo "dir ${outputDir}" >> ${doneFile}
ab946185 843 if summarizeLogs >> ${doneFile}; then
844 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
845 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
846 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
847 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
848 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
922d0b10 849 else
850 if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
851 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
852 fi
853 if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
854 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
855 fi
ab946185 856 fi
71353a5f 857
858 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 859 return 0
78d0bbfd 860)
cb473870 861
862goMerge()
78d0bbfd 863(
cb473870 864 #generic root merge using CPass1 merge.C script
71353a5f 865 inputList=${1}
866 outputFile=${2}
0eaf38c7 867 configFile=${3-"benchmark.config"}
71353a5f 868 shift 3
6fc48f9e 869 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 870
871 #record the working directory provided by the batch system
872 batchWorkingDirectory=${PWD}
fc7e1e45 873
71353a5f 874 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
875 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
876 umask 0002
877 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
878 rm -f ${outputFile}
879 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 880 return 0
78d0bbfd 881)
fc7e1e45 882
cb473870 883goSubmitMakeflow()
5dfac970 884{
cb473870 885 #run
71353a5f 886 productionID=${1}
887 inputFileList=${2}
888 configFile=${3}
889 shift 3
444b9916 890 extraOpts=("$@")
6fc48f9e 891 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 892
893 #record the working directory provided by the batch system
894 batchWorkingDirectory=${PWD}
fc7e1e45 895
5dfac970 896 [[ -z ${configFile} ]] && configFile="benchmark.config"
897 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 898
71353a5f 899 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 900 echo "setting the makflow path from the config: "
901 echo " export PATH=${makeflowPath}:${PATH}"
902 export PATH=${makeflowPath}:${PATH}
903 fi
904
39527a15 905 #create the common output dir and the meta dir
906 commonOutputPath=${baseOutputDirectory}/${productionID}
907 if [[ -d ${commonOutputPath} ]]; then
908 echo "output dir ${commonOutputPath} exists!"
0eaf38c7 909 #return 1
39527a15 910 else
911 mkdir -p ${commonOutputPath}
912 fi
913 mkdir -p ${commonOutputPath}/meta
839b07ca 914
0eaf38c7 915 self=${0}
916 #if which greadlink; then self=$(greadlink -f "${0}"); fi
839b07ca 917
0eaf38c7 918 #for reference copy the setup to the output dir
839b07ca 919 cp ${self} ${commonOutputPath}
920 cp ${configFile} ${commonOutputPath}
921 cp ${inputList} ${commonOutputPath}
39527a15 922
cb473870 923 #submit - use makeflow if available, fall back to old stuff when makeflow not there
924 if which makeflow; then
cb473870 925
39527a15 926 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
71353a5f 927
cb473870 928 makeflow ${makeflowOptions} benchmark.makeflow
929 cd ../
930 else
931 echo "no makeflow!"
932 fi
5de3064b 933 return 0
5dfac970 934}
fc7e1e45 935
5dfac970 936goGenerateMakeflow()
78d0bbfd 937(
5dfac970 938 #generate the makeflow file
ab946185 939 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 940 productionID=${1}
941 inputFileList=${2}
942 configFile=${3}
943 shift 3
444b9916 944 extraOpts=("$@")
0eaf38c7 945
946 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
947 for (( i=0;i<${#extraOpts[@]};i++ )); do
948 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
949 done
950 extraOpts+=("encodedSpaces=1")
951
39527a15 952 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
953
1a0e38e3 954 #extra safety
955 if [[ -z ${commonOutputPath} ]]; then
956 commonOutputPath=${baseOutputDirectory}/${productionID}
957 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
958 fi
bff71da0 959
39527a15 960 #record the working directory provided by the batch system
961 batchWorkingDirectory=${PWD}
fc7e1e45 962
5dfac970 963 [[ -z ${configFile} ]] && configFile="benchmark.config"
964 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 965
5dfac970 966 #these files will be made a dependency - will be copied to the working dir of the jobs
967 declare -a copyFiles
968 inputFiles=(
969 "OCDB.root"
970 "localOCDBaccessConfig.C"
cb473870 971 "QAtrain_duo.C"
972 "runCPass1.sh"
973 "recCPass1.C"
974 "recCPass1_OuterDet.C"
975 "runCalibTrain.C"
976 "runCPass0.sh"
977 "recCPass0.C"
5dfac970 978 )
979 for file in ${inputFiles[*]}; do
980 [[ -f ${file} ]] && copyFiles+=("${file}")
981 done
982
983 #create the makeflow file
cb473870 984 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 985 declare -A arr_cpass0_merged arr_cpass1_merged
986 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
987 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 988 declare -A arr_cpass0_profiled_outputs
444b9916 989 declare -A listOfRuns
990 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 991 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 992 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 993 [[ -z ${runNumber} ]] && continue
fc7e1e45 994 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 995
444b9916 996 unset arr_cpass0_outputs
997 unset arr_cpass1_outputs
998 declare -a arr_cpass0_outputs
999 declare -a arr_cpass1_outputs
fc7e1e45 1000
cb473870 1001 jobindex=0
6fc48f9e 1002 inputFile=""
5dfac970 1003 while read inputFile; do
1004 currentDefaultOCDB=${defaultOCDB}
cb473870 1005 [[ -z ${autoOCDB} ]] && autoOCDB=1
1006 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1007 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 1008 fi
fc7e1e45 1009
5dfac970 1010 #CPass0
39527a15 1011 arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
444b9916 1012 echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1013 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
5dfac970 1014
1015 #CPass1
39527a15 1016 arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1017 echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
0eaf38c7 1018 echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
5dfac970 1019 ((jobindex++))
fc7e1e45 1020
71353a5f 1021 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 1022
5dfac970 1023 #CPass0 list of Calib files to merge
39527a15 1024 arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
444b9916 1025 echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1026 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
71353a5f 1027 echo
1028
1029 #CPass0 merging
39527a15 1030 arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1031 echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
0eaf38c7 1032 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
fc7e1e45 1033
ab946185 1034 #CPass1 list of Calib/QA/ESD/filtered files
1035 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1036 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 1037 # this corresponds to the last merging stage)
39527a15 1038 arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
444b9916 1039 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1040 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
5dfac970 1041 echo
39527a15 1042 arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
ab946185 1043 echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1044 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1045 echo
39527a15 1046 arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
444b9916 1047 echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1048 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1049 echo
39527a15 1050 arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
444b9916 1051 echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1052 echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1053 echo
444b9916 1054
71353a5f 1055 #CPass1 merging
39527a15 1056 arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1057 echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
0eaf38c7 1058 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
fc7e1e45 1059
6fc48f9e 1060 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1061 if [[ -n ${profilingCommand} ]]; then
0eaf38c7 1062 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1063 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1064 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1065 jobindex="profiling"
1066
1067 arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
6fc48f9e 1068 echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1069 profilingCommand=$(encSpaces "${profilingCommand}")
1070 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
6fc48f9e 1071 fi
1072
444b9916 1073 done #runs
fc7e1e45 1074
5dfac970 1075 #Summary
1a0e38e3 1076 echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
0eaf38c7 1077 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
cb473870 1078 echo
1079
5de3064b 1080 return 0
78d0bbfd 1081)
cb473870 1082
1083goPrintValues()
78d0bbfd 1084(
cb473870 1085 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1086 if [[ $# -lt 3 ]]; then
1087 echo "goPrintValues key outputFile inputFiles"
1088 echo "if outputFile is \"-\" dont redirect to a file"
1089 return
1090 fi
1091 key=${1}
1092 outputFile=${2}
1093 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1094 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1095 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1096 return 0
78d0bbfd 1097)
fc7e1e45 1098
5dfac970 1099goCreateQAplots()
78d0bbfd 1100(
5dfac970 1101 umask 0002
71353a5f 1102 mergedQAfileList=${1}
1103 productionID=${2}
1104 outputDir=${3}
1105 configFile=${4}
1106 shift 4
6fc48f9e 1107 if ! parseConfig ${configFile} ${@}; then return 1; fi
71353a5f 1108
39527a15 1109 #record the working directory provided by the batch system
1110 batchWorkingDirectory=${PWD}
1111
71353a5f 1112 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1113
71353a5f 1114 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
0eaf38c7 1115 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
71353a5f 1116 echo "${0} $*"
1117
1118 olddir=${PWD}
1119 mkdir -p ${outputDir}
1120 cd ${outputDir}
839b07ca 1121 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1122
9510565c 1123 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
fc7e1e45 1124
71353a5f 1125 cd ${olddir}
5de3064b 1126 return 0
78d0bbfd 1127)
fc7e1e45 1128
5dfac970 1129goTest()
78d0bbfd 1130(
1131 echo AA
1132)
fc7e1e45 1133
5dfac970 1134alirootInfo()
78d0bbfd 1135(
5dfac970 1136 umask 0002
71353a5f 1137 # save aliroot repository info
1138 [[ -z "${ALICE_ROOT}" ]] && return 1
1139
1140 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1141 echo "\${ROOTSYS}=${ROOTSYS}"
1142 echo "\${PATH}=${PATH}"
1143 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
0eaf38c7 1144 echo
71353a5f 1145
1146 pushd ${PWD}
1147 cd ${ALICE_ROOT}
71353a5f 1148
1149 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1150 git status
5dfac970 1151 echo ""
1152 echo ""
71353a5f 1153 git diff ${currentBranch}
1154 popd
5de3064b 1155 return 0
78d0bbfd 1156)
fc7e1e45 1157
5dfac970 1158setYear()
78d0bbfd 1159(
5dfac970 1160 #set the year
71353a5f 1161 # ${1} - year to be set
1162 # ${2} - where to set the year
1163 year1=$(guessYear ${1})
1164 year2=$(guessYear ${2})
1165 local path=${2}
1166 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1167 echo ${path}
5de3064b 1168 return 0
78d0bbfd 1169)
fc7e1e45 1170
5dfac970 1171guessPeriod()
78d0bbfd 1172(
5dfac970 1173 #guess the period from the path, pick the rightmost one
1174 local IFS="/"
71353a5f 1175 declare -a path=( ${1} )
5dfac970 1176 local dirDepth=${#path[*]}
1177 for ((x=${dirDepth}-1;x>=0;x--)); do
1178 local field=${path[${x}]}
71353a5f 1179 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1180 done
71353a5f 1181 echo ${period}
5de3064b 1182 return 0
78d0bbfd 1183)
fc7e1e45 1184
5dfac970 1185guessYear()
78d0bbfd 1186(
5dfac970 1187 #guess the year from the path, pick the rightmost one
1188 local IFS="/"
71353a5f 1189 declare -a path=( ${1} )
5dfac970 1190 local dirDepth=${#path[*]}
1191 for ((x=${dirDepth}-1;x>=0;x--)); do
1192 local field=${path[${x}]}
71353a5f 1193 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1194 done
71353a5f 1195 echo ${year}
5de3064b 1196 return 0
78d0bbfd 1197)
fc7e1e45 1198
5dfac970 1199guessRunNumber()
78d0bbfd 1200(
5dfac970 1201 #guess the run number from the path, pick the rightmost one
71353a5f 1202 #works for /path/foo/000123456/bar/...
1203 #and /path/foo.run123456.bar
1204 local IFS="/."
1205 declare -a path=( ${1} )
5dfac970 1206 local dirDepth=${#path[*]}
1207 for ((x=${dirDepth}-1;x>=0;x--)); do
1208 local field=${path[${x}]}
71353a5f 1209 field=${field/run/000}
617f8886 1210 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1211 done
71353a5f 1212 echo ${runNumber}
5de3064b 1213 return 0
78d0bbfd 1214)
fc7e1e45 1215
cb473870 1216validateLog()
78d0bbfd 1217(
71353a5f 1218 log=${1}
cb473870 1219 errorConditions=(
71353a5f 1220 'There was a crash'
1221 'floating'
1222 'error while loading shared libraries'
1223 'std::bad_alloc'
1224 's_err_syswatch_'
1225 'Thread [0-9]* (Thread'
1226 'AliFatal'
1227 'core dumped'
1228 '\.C.*error:.*\.h: No such file'
839b07ca 1229 'line.*Aborted'
71353a5f 1230 )
1231
1232 warningConditions=(
1233 'This is serious !'
1234 'rocVoltage out of range:'
cb473870 1235 )
1236
71353a5f 1237 local logstatus=0
1238 local errorSummary=""
1239 local warningSummary=""
1240
1241 for ((i=0; i<${#errorConditions[@]};i++)); do
1242 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1243 [[ -n ${tmp} ]] && tmp+=" : "
1244 errorSummary+=${tmp}
1245 done
1246
1247 for ((i=0; i<${#warningConditions[@]};i++)); do
1248 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1249 [[ -n ${tmp} ]] && tmp+=" : "
1250 warningSummary+=${tmp}
1251 done
1252
1253 if [[ -n ${errorSummary} ]]; then
1254 echo "${errorSummary}"
1255 return 1
1256 fi
cb473870 1257
71353a5f 1258 if [[ -n ${warningSummary} ]]; then
1259 echo "${warningSummary}"
1260 return 2
1261 fi
1262
1263 return 0
78d0bbfd 1264)
cb473870 1265
5dfac970 1266summarizeLogs()
78d0bbfd 1267(
5dfac970 1268 #print a summary of logs
1269 logFiles=(
1270 "*.log"
cb473870 1271 "stdout"
1272 "stderr"
5dfac970 1273 )
fc7e1e45 1274
cb473870 1275 #check logs
71353a5f 1276 local logstatus=0
5dfac970 1277 for log in ${logFiles[*]}; do
71353a5f 1278 finallog=${PWD%/}/${log}
1279 [[ ! -f ${log} ]] && continue
1280 errorSummary=$(validateLog ${log})
1281 validationStatus=$?
1282 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1283 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1284 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1285 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1286 echo "${finallog} OK"
fc7e1e45 1287 else
71353a5f 1288 echo "${finallog} OK"
fc7e1e45 1289 fi
71353a5f 1290 elif [[ ${validationStatus} -eq 1 ]]; then
1291 echo "${finallog} BAD ${errorSummary}"
ab946185 1292 logstatus=1
71353a5f 1293 elif [[ ${validationStatus} -eq 2 ]]; then
1294 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1295 fi
1296 done
cb473870 1297
1298 #report core files
71353a5f 1299 while read x; do
1300 echo ${x}
1301 chmod 644 ${x}
1302 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1303 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1304
71353a5f 1305 return ${logstatus}
78d0bbfd 1306)
fc7e1e45 1307
5dfac970 1308spitOutLocalOCDBaccessConfig()
1309{
1310 umask 0002
71353a5f 1311 #find ${1} -name "*root" | \
1312 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1313 while read line
1314 do
71353a5f 1315 local tmp=${line#${1}}
5dfac970 1316 echo ${tmp%/*} | \
71353a5f 1317 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1318 done
5de3064b 1319 return 0
5dfac970 1320}
fc7e1e45 1321
71353a5f 1322goMakeLocalOCDBaccessConfig()
5dfac970 1323{
1324 umask 0002
1325 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1326 local localOCDBpathCPass0=${1}
1327 local OCDBpathPrefix=${2}
1328 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1329
1330 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1331 tar xzf ${localOCDBpathCPass0}
1332 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1333 fi
fc7e1e45 1334
5dfac970 1335 echo
1336 echo creating the specific storage script
1337 echo localOCDBaccessConfig.C
71353a5f 1338 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1339 echo
fc7e1e45 1340
5dfac970 1341 local tempLocalOCDB=""
1342 if [[ -f localOCDBaccessConfig.C ]]; then
0eaf38c7 1343 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
71353a5f 1344 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1345 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1346 fi
fc7e1e45 1347
5dfac970 1348 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1349 echo "{" >> localOCDBaccessConfig.C
1350 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1351 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1352 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1353 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1354
71353a5f 1355 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1356
5dfac970 1357 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1358 echo
1359 echo "!!!!!!! CPass0 produced no OCDB entries"
1360 return 1
1361 fi
5de3064b 1362 return 0
5dfac970 1363}
fc7e1e45 1364
cb473870 1365goMakeFilteredTrees()
78d0bbfd 1366(
71353a5f 1367 outputDir=${1}
1368 runNumber=${2}
cb473870 1369 #get path to input list
71353a5f 1370 inputListfiles=${3}
cb473870 1371 #get scale number for tracks
71353a5f 1372 filterT=${4}
cb473870 1373 #get scale number for V0s
71353a5f 1374 filterV=${5}
cb473870 1375 #get OCDB path (optional)
1376 OCDBpath=${6}
1377 #get max number of files
1378 maxFiles=${7-"1000000"}
1379 #get offset of first file
1380 offsetFile=${8-"0"}
1381 #get max number of events
1382 maxEvents=${9-"30000000"}
1383 #get offset of first event
1384 offsetEvent=${10-"0"}
1385 configFile=${11-"benchmark.config"}
1386 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1387 shift 12
6fc48f9e 1388 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1389
1390 #record the working directory provided by the batch system
1391 batchWorkingDirectory=${PWD}
1392
78d0bbfd 1393 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
71353a5f 1394 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1395
cb473870 1396 cat > filtering.log << EOF
1397 goMakeFilteredTrees config:
71353a5f 1398 runpath=${runpath}
1399 outputDir=${outputDir}
1400 commonOutputPath=${commonOutputPath}
1401 ALICE_ROOT=${ALICE_ROOT}
1402 PATH=${PATH}
1403 offsetEvent=$offsetEvent
1404 configFile=$configFile
1405 esdFileName=$esdFileName
fab31805 1406 inputListfiles=$inputListfiles
1407 doneFile=$doneFile
cb473870 1408EOF
1409
fab31805 1410 #runpath=${outputDir}
1411 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1412 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1413 #mkdir -p ${outputDir}
1414 #mkdir -p ${runpath}
1415 #if ! cd ${runpath}; then
1416 # echo "PWD=$PWD is not the runpath=${runpath}"
1417 # touch ${doneFile}
1418 # return 1
1419 #fi
1420
cb473870 1421 if [[ -z ${pretend} ]];then
71353a5f 1422 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1423 else
0eaf38c7 1424 sleep ${pretendDelay}
71353a5f 1425 touch filtering.log FilterEvents_Trees.root
cb473870 1426 fi
1427 pwd
71353a5f 1428 /bin/ls
1429 touch ${doneFile}
1430 summarizeLogs >> ${doneFile}
fab31805 1431
1432 #echo mv -f * ${outputDir}
1433 #mv -f * ${outputDir}
1434 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1435 #cd ${commonOutputPath}
1436 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1437
5de3064b 1438 return 0
78d0bbfd 1439)
cb473870 1440
1441submit()
1442{
1443 umask 0002
444b9916 1444 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1445 JobID=${1}
1446 startID=${2}
1447 endID=${3}
1448 waitForJOBID=${4}
1449 command=${5}
444b9916 1450 shift 5
78d0bbfd 1451 local commandArgs=("$@")
444b9916 1452
1453 #add quote strings around the extra arguments
1454 for ((i=0; i<${#commandArgs[@]}; i++)); do
1455 commandArgs[i]=\"${commandArgs[i]}\"
1456 done
cb473870 1457
71353a5f 1458 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1459
1460 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1461
1462 batchSystem="SGE"
1463
71353a5f 1464 if [[ -z "${newFarm}" ]]
cb473870 1465 then
1466 #old LSF
1467 # submit it (as job array)
71353a5f 1468 nFiles=$(( ${endID}-${startID}+1 ))
1469 while [ ${startID} -le ${nFiles} ] ; do
1470 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1471 endID=$(expr ${startID} + 999)
cb473870 1472 else
71353a5f 1473 endID=${nFiles}
cb473870 1474 fi
1475 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1476 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1477 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1478 else
71353a5f 1479 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1480 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1481 fi
71353a5f 1482 startID=$(expr ${endID} + 1)
cb473870 1483 done
1484 else
1485 #new SGE farm
1486 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1487 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1488 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1489 else
444b9916 1490 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1491 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1492 fi
1493 fi
5de3064b 1494 return 0
cb473870 1495}
1496
1497goSubmitBatch()
5dfac970 1498{
71353a5f 1499 if [[ $# -lt 3 ]]; then
1500 echo "minimal use:"
1501 echo " ${0} submit fileList productionID configFile"
1502 return 0
1503 fi
1504
1505 productionID=${1}
1506 inputList=${2}
1507 configFile=${3:-"benchmark.config"}
0eaf38c7 1508 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
71353a5f 1509 shift 3
444b9916 1510 extraOpts=("$@")
6fc48f9e 1511 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1512
b0e401b4 1513 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1514 for (( i=0;i<${#extraOpts[@]};i++ )); do
1515 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1516 done
1517 extraOpts+=("encodedSpaces=1")
1518
39527a15 1519 #record the working directory provided by the batch system
1520 batchWorkingDirectory=${PWD}
fc7e1e45 1521
5dfac970 1522 #redirect all output to submit.log
1523 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1524 exec 7>&1
1525 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1526
5dfac970 1527 umask 0002
71353a5f 1528 echo ${0}" submit $*"
1529 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1530 then
fc7e1e45 1531 echo
71353a5f 1532 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1533 echo
71353a5f 1534 return
fc7e1e45 1535 fi
fc7e1e45 1536
5dfac970 1537 # check if config file is there
71353a5f 1538 if [ ! -f ${configFile} ]; then
1539 echo "ERROR! Config File '${configFile}' not found" >&2
1540 return
5dfac970 1541 else
71353a5f 1542 echo "Using Config File: '${configFile}'"
fc7e1e45 1543 fi
1544
9510565c 1545 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1546
71353a5f 1547 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
0eaf38c7 1548 self=${0}
1549 #if which greadlink; then self=$(greadlink -f "${0}"); fi
71353a5f 1550 configPath=$(dirname ${configFile})
5dfac970 1551 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1552
cb473870 1553 mkdir -p ${commonOutputPath}
71353a5f 1554 mkdir -p ${commonOutputPath}/logs
1555 mkdir -p ${commonOutputPath}/meta
1556
1557 cp ${self} ${commonOutputPath}
1558 cp ${configFile} ${commonOutputPath}
1559 cp ${inputList} ${commonOutputPath}
cb473870 1560 self=${commonOutputPath}/${self##*/}
71353a5f 1561 chmod u+x ${self}
cb473870 1562 configFile=${commonOutputPath}/${configFile##*/}
1563 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1564
5dfac970 1565 #convert to absolut pathnames
0eaf38c7 1566 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
5dfac970 1567 #make list of runs
71353a5f 1568 if [[ -z ${runNumber} ]]; then
1569 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1570 else
71353a5f 1571 listOfRuns=${runNumber}
5dfac970 1572 fi
fc7e1e45 1573
0eaf38c7 1574 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
fc7e1e45 1575
5dfac970 1576 echo ""
1577 echo "### BEGIN CONFIGURATION ###"
1578 echo ""
1579 echo "GENERAL:"
1580 echo ""
71353a5f 1581 echo " productionID: ${productionID}"
1582 echo " batchCommand: ${batchCommand}"
1583 echo " batchFlags: ${batchFlags}"
9510565c 1584 echo " alirootEnv: ${alirootEnv}"
1585 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1586 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1587 echo " inputList: ${inputList}"
1588 echo " configPath: ${configPath}"
1589 echo " commonOutputPath: ${commonOutputPath}"
1590 echo " defaultOCDB: ${defaultOCDB}"
1591 echo " autoOCDB: ${autoOCDB}"
1592 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1593 echo " runs:"
1594 echo " ${listOfRuns[*]}"
1595 echo ""
1596 echo "THE TRAIN WILL RUN:"
fc7e1e45 1597
71353a5f 1598 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1599 echo " Pass0 - Recontruction"
1600 fi
fc7e1e45 1601
71353a5f 1602 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1603 echo " Pass0 - merging and OCDB export"
1604 fi
fc7e1e45 1605
71353a5f 1606 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1607 echo " Pass1 - Recontruction"
1608 fi
71353a5f 1609 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1610 echo " Pass1 - merging and OCDB export"
1611 fi
fc7e1e45 1612
5dfac970 1613 echo ""
1614 echo "LIMITS:"
71353a5f 1615 echo " max. Events/Chunk: ${nEvents}"
1616 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1617 echo ""
1618 echo "### END CONFIGURATION ###"
1619 echo ""
fc7e1e45 1620
36b5de1a 1621
5dfac970 1622 # check if input file is there
71353a5f 1623 if [ ! -f ${inputList} ]; then
1624 echo "ERROR! Input List '${inputList}' not found" >&2
1625 return
5dfac970 1626 fi
1627
1628 # define jobid (for dependent jobs)
cb473870 1629 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1630 #for each run we submit one jobarray:
fc7e1e45 1631 for runNumber in ${listOfRuns[*]}; do
cb473870 1632
71353a5f 1633 [[ -z ${runNumber} ]] && continue
fc7e1e45 1634 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1635
1636 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1637 JOBID1="p0_${JOBpostfix}"
1638 JOBID1wait="w0_${JOBpostfix}"
1639 JOBID2="m0_${JOBpostfix}"
1640 JOBID2wait="wm0_${JOBpostfix}"
1641 JOBID3="op0_${JOBpostfix}"
1642 JOBID3wait="wop0_${JOBpostfix}"
1643 JOBID4="p1_${JOBpostfix}"
1644 JOBID4wait="w1_${JOBpostfix}"
1645 JOBID5="m1_${JOBpostfix}"
1646 JOBID5wait="wm1_${JOBpostfix}"
1647 JOBID6="s1_${JOBpostfix}"
1648 JOBID6wait="ws1_${JOBpostfix}"
1649 JOBID7="QA_${JOBpostfix}"
1650 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1651 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1652 LASTJOB="000"
1653
0eaf38c7 1654 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
cb473870 1655
1656 currentDefaultOCDB=${defaultOCDB}
1657 [[ -z ${autoOCDB} ]] && autoOCDB=1
1658 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1659 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1660 fi
1661
71353a5f 1662 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1663
1664 ###############################################################################
1665 #run one chunk with valgrind:
6fc48f9e 1666 if [[ -n ${profilingCommand} ]]; then
1667 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1668 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1669 submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
71353a5f 1670 fi
fc7e1e45 1671
5dfac970 1672 ################################################################################
1673 ################################################################################
1674 # run the CPass0 if requested
fc7e1e45 1675
71353a5f 1676 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1677
fc7e1e45 1678 echo
71353a5f 1679 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1680 echo
fc7e1e45 1681
5dfac970 1682 # create directory and copy all files that are needed
cb473870 1683 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1684 mkdir -p ${targetDirectory}
1685
cb473870 1686 filesCPass0=(
71353a5f 1687 "${configPath}/runCPass0.sh"
1688 "${configPath}/recCPass0.C"
1689 "${configPath}/runCalibTrain.C"
1690 "${configPath}/localOCDBaccessConfig.C"
1691 "${configPath}/OCDB*.root"
1692 "${configPath}/sim.C"
1693 "${configPath}/rec.C"
1694 "${configPath}/Config.C"
cb473870 1695 )
1696 for file in ${filesCPass0[*]}; do
1697 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1698 done
fc7e1e45 1699
71353a5f 1700 localInputList=${targetDirectory}/${inputList##*/}
1701 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1702 # limit nFiles to nMaxChunks
71353a5f 1703 nFiles=$(wc -l < ${localInputList})
1704 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1705 echo "raw files in list: ${nFiles}"
1706 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1707 nFiles=${nMaxChunks}
5dfac970 1708 fi
71353a5f 1709 echo "raw files to process: ${nFiles}"
1710 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1711 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1712 nFilesToWaitFor=${nFiles}
5dfac970 1713 else
71353a5f 1714 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1715 fi
71353a5f 1716 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1717 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1718
444b9916 1719 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1720
5dfac970 1721 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
444b9916 1722 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
71353a5f 1723 LASTJOB=${JOBID1wait}
fc7e1e45 1724
5dfac970 1725 fi #end running CPass0
1726 ################################################################################
fc7e1e45 1727
fc7e1e45 1728
5dfac970 1729 ################################################################################
1730 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1731
71353a5f 1732 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1733
5dfac970 1734 echo
71353a5f 1735 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1736 echo
fc7e1e45 1737
cb473870 1738 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1739 mkdir -p ${targetDirectory}
fc7e1e45 1740
cb473870 1741 #copy the scripts
1742 filesMergeCPass0=(
71353a5f 1743 "${configPath}/OCDB.root"
1744 "${configPath}/mergeMakeOCDB.byComponent.sh"
1745 "${configPath}/mergeMakeOCDB.sh"
1746 "${configPath}/localOCDBaccessConfig.C"
1747 "${configPath}/mergeByComponent.C"
1748 "${configPath}/makeOCDB.C"
1749 "${configPath}/merge.C"
cb473870 1750 )
1751 for file in ${filesMergeCPass0[*]}; do
1752 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1753 done
1754
444b9916 1755 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1756 LASTJOB=${JOBID2}
fc7e1e45 1757
71353a5f 1758 if [[ -n ${generateMC} ]]; then
444b9916 1759 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1760 LASTJOB="mrl${JOBpostfix}"
1761 fi
fc7e1e45 1762
5dfac970 1763 echo
fc7e1e45 1764 fi
5dfac970 1765 # end of merging CPass0
1766 ################################################################################
fc7e1e45 1767
5dfac970 1768 ################################################################################
1769 ################################################################################
1770 # run the CPass1 if requested
fc7e1e45 1771
71353a5f 1772 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1773
cb473870 1774 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1775 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1776
5dfac970 1777 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1778 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1779
5dfac970 1780 echo
71353a5f 1781 echo "starting CPass1... for run ${runNumber}"
5dfac970 1782 echo
1783
1784 # create directory and copy all files that are needed
71353a5f 1785 mkdir -p ${targetDirectory}
5dfac970 1786
cb473870 1787 filesCPass1=(
71353a5f 1788 "${configPath}/runCPass1.sh"
1789 "${configPath}/recCPass1.C"
1790 "${configPath}/recCPass1_OuterDet.C"
1791 "${configPath}/runCalibTrain.C"
1792 "${configPath}/QAtrain_duo.C"
1793 "${configPath}/localOCDBaccessConfig.C"
1794 "${configPath}/OCDB.root"
cb473870 1795 )
1796 for file in ${filesCPass1[*]}; do
1797 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1798 done
5dfac970 1799
71353a5f 1800 if [[ -n ${generateMC} ]]; then
1801 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1802 else
1803 localInputList=${targetDirectory}/${inputList##*/}
1804 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1805 fi
5dfac970 1806 # limit nFiles to nMaxChunks
71353a5f 1807 nFiles=$(wc -l < ${localInputList})
1808 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1809 echo "raw files in list: ${nFiles}"
1810 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1811 nFiles=${nMaxChunks}
5dfac970 1812 fi
71353a5f 1813 echo "raw files to process: ${nFiles}"
1814 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1815 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1816 nFilesToWaitFor=${nFiles}
fc7e1e45 1817 else
71353a5f 1818 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 1819 fi
71353a5f 1820 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1821 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1822
444b9916 1823 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1824
5dfac970 1825 ################################################################################
1826 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
444b9916 1827 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
71353a5f 1828 LASTJOB=${JOBID4wait}
5dfac970 1829 ################################################################################
fc7e1e45 1830
5dfac970 1831 echo
1832 fi #end running CPass1
fc7e1e45 1833
5dfac970 1834 ################################################################################
1835 # submit merging of CPass1, depends on the reconstruction
71353a5f 1836 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1837
5dfac970 1838 echo
71353a5f 1839 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 1840 echo
fc7e1e45 1841
cb473870 1842 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1843 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1844 mkdir -p ${targetDirectory}
fc7e1e45 1845
cb473870 1846 # copy files
1847 filesMergeCPass1=(
71353a5f 1848 "${configPath}/OCDB.root"
1849 "${configPath}/localOCDBaccessConfig.C"
1850 "${configPath}/mergeMakeOCDB.byComponent.sh"
1851 "${configPath}/mergeByComponent.C"
1852 "${configPath}/makeOCDB.C"
1853 "${configPath}/merge.C"
1854 "${configPath}/mergeMakeOCDB.sh"
1855 "${configPath}/QAtrain_duo.C"
cb473870 1856 )
1857 for file in ${filesMergeCPass1[*]}; do
1858 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1859 done
1860
444b9916 1861 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1862 LASTJOB=${JOBID5}
5dfac970 1863 echo
1864 fi
1865
444b9916 1866 ###############################
1867 #if [ ${runESDfiltering} -eq 1 ]; then
1868 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1869 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1870 # echo
1871 # echo submitting filtering for run ${runNumber}
1872 # echo
1873 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1874 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1875 # LASTJOB=${JOBfilterESDcpass1}
1876 #fi
cb473870 1877
5dfac970 1878 done
1879
71353a5f 1880 #################################################################################
1881 #################################################################################
444b9916 1882 #if [ ${runESDfiltering} -eq 1 ]; then
1883 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1884 #else
1885 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1886 #fi
71353a5f 1887 LASTJOB=${JOBID5wait}
cb473870 1888
5de3064b 1889 #################################################################################
1890 echo
1891 echo "submit make a summary"
1892 echo
5dfac970 1893
444b9916 1894 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
5de3064b 1895 LASTJOB=${JOBID6}
71353a5f 1896 #################################################################################
5dfac970 1897
5dfac970 1898 #restore stdout
1899 exec 1>&7 7>&-
1900 echo "jobs submitted."
5de3064b 1901 return 0
fc7e1e45 1902}
1903
cb473870 1904goWaitForOutput()
78d0bbfd 1905(
cb473870 1906 umask 0002
71353a5f 1907 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1908 echo searchPath=${1}
1909 echo fileName=${2}
1910 echo numberOfFiles=${3}
1911 echo maxSecondsToWait=${4}
1912 searchPath=${1}
1913 fileName=${2}
1914 numberOfFiles=${3}
1915 maxSecondsToWait=${4}
1916 extraFindOptions=${5}
1917 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1918 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 1919 while true; do
71353a5f 1920 #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
cb473870 1921 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 1922 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1923 [[ ${n} -ge ${numberOfFiles} ]] && break
1924 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 1925 sleep 60
1926 done
1927 echo "DONE! exiting..."
5de3064b 1928 return 0
78d0bbfd 1929)
cb473870 1930
71353a5f 1931mergeSysLogs()
78d0bbfd 1932(
71353a5f 1933 outputFile=${1}
1934 shift
1935 inputFiles="$@"
1936 i=0
1937 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1938 while read x; do
1939 runNumber=$(guessRunNumber ${x})
1940 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1941 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1942 (( i++ ))
1943 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 1944 return 0
78d0bbfd 1945)
71353a5f 1946
1947goMakeMergedSummaryTree()
78d0bbfd 1948(
71353a5f 1949 # create list of calibration entries
1950 # takes no arguments, just run it in the base output
1951 # directory with the following files in the working directory
1952 #
1953 # Calibration file lists:
1954 # cpass0.dcsTree.list, cpass1.dcsTree.list
1955 # QA trending root file:
1956 # trending.root
1957 #
1958 # Production infoo ascii files:
1959 # summary_pass0.tree
1960 # summary_pass1.tree
1961 #
617f8886 1962
71353a5f 1963 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1964 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1965 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1966 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1967 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1968
1969 #first, dump the C macro to file
1970 cat << EOF > mergeTree.C
1971 //
1972 // Merge summary information
1973 // Following files are expected to be in the working directory
1974 //
1975 // Calibration file lists:
1976 // cpass0.dcsTree.list, cpass1.dcsTree.list
1977 // QA trending root file:
1978 // trending.root
1979 //
1980 // Production infoo ascii files:
1981 // summary_pass0.tree
1982 // summary_pass1.tree
1983 //
1984 void mergeTree(){
1985 //
1986 //
1987 //
1988 // Calibration values dump
1989 //
839b07ca 1990 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 1991 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
839b07ca 1992 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 1993 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1994 //
1995 // Calibration status dump
1996 //
1997 TFile *fprod = TFile::Open("fproduction.root","recreate");
1998 TTree tree0, tree1;
839b07ca 1999 //Printf("reading summary_pass0.tree");
71353a5f 2000 tree0.ReadFile("summary_pass0.tree");
839b07ca 2001 //Printf("reading summary_pass1.tree");
71353a5f 2002 tree1.ReadFile("summary_pass1.tree");
2003 tree0.Write("CPass0");
2004 tree1.Write("CPass1");
2005 fprod->Close();
2006 //
2007 //
2008 //
2009 TString stringSetup="";
2010 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2011 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2012 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2013 //
2014 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2015 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2016 //
839b07ca 2017 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 2018 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2019 }
2020EOF
2021
2022 aliroot -b -q "mergeTree.C" > mergeTrees.log
2023 return $?
78d0bbfd 2024)
71353a5f 2025
2026goMakeSummary()
78d0bbfd 2027(
71353a5f 2028 #all the final stuff goes in here for ease of use:
2029 # summary logs
2030 # qa plot making
2031 # final file lists
444b9916 2032 #some defaults:
2033 log="summary.log"
2034 productionID="qa"
71353a5f 2035
2036 configFile=${1}
2037 shift 1
444b9916 2038 extraOpts=("$@")
6fc48f9e 2039 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 2040
0eaf38c7 2041 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
839b07ca 2042
39527a15 2043 #record the working directory provided by the batch system
2044 batchWorkingDirectory=${PWD}
2045
71353a5f 2046 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2047
71353a5f 2048 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2049
2050 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2051
39527a15 2052 #copy some useful stuff
2053 #and go to the commonOutputPath
2054 cp ${configFile} ${commonOutputPath}
2055 cd ${commonOutputPath}
2056
2057 exec &> >(tee ${log})
2058
71353a5f 2059 #summarize the global stuff
2060 echo "env script: ${alirootSource} ${alirootEnv}"
2061 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2062 echo "commonOutputPath=${commonOutputPath}"
2063
2064 #summarize the stacktraces
2065 awk '
2066 BEGIN {
2067 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2068 RS="#[0-9]*";
2069 aliroot=0;
2070 }
2071 {
2072 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2073 gsub("#","",RT);
2074 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2075 }
2076 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2077 awk '
2078 BEGIN {
2079 RS="#[0-9]*";
2080 aliroot=0;
2081 }
2082 {
2083 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2084 gsub("#","",RT);
2085 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2086 }
2087 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2088
2089 echo total numbers for the production:
2090 echo
2091 awk 'BEGIN {nFiles=0;nCore=0;}
2092 /^calibfile/ {nFiles++;}
2093 /core dumped/ {nCore++i;}
2094 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2095 awk 'BEGIN {nOK=0; nBAD=0; }
2096 /\/rec.log OK/ {nOK++;}
2097 /\/rec.log BAD/ {nBAD++;}
2098 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2099 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2100 awk 'BEGIN {nOK=0; nBAD=0; }
2101 /\/calib.log OK/ {nOK++;}
2102 /\/calib.log BAD/ {nBAD++;}
2103 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2104
2105 awk 'BEGIN {nOK=0; nBAD=0; }
2106 /merge.log OK/ {nOK++;}
2107 /merge.log BAD/ {nBAD++;}
2108 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2109 awk 'BEGIN {nOK=0; nBAD=0; }
2110 /ocdb.log OK/ {nOK++;}
2111 /ocdb.log BAD/ {nBAD++;}
2112 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2113
2114 echo
2115 awk 'BEGIN {nFiles=0;nCore=0;}
2116 /^calibfile/ {nFiles++;}
2117 /core dumped/ {nCore++;}
2118 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2119 awk 'BEGIN {nOK=0; nBAD=0; }
2120 /\/rec.log OK/ {nOK++;}
2121 /\/rec.log BAD/ {nBAD++;}
2122 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2123 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2124 awk 'BEGIN {nOK=0; nBAD=0; }
2125 /\/calib.log OK/ {nOK++;}
2126 /\/calib.log BAD/ {nBAD++;}
2127 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2128
2129 awk 'BEGIN {nOK=0; nBAD=0; }
2130 /merge.log OK/ {nOK++;}
2131 /merge.log BAD/ {nBAD++;}
2132 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2133 awk 'BEGIN {nOK=0; nBAD=0; }
2134 /ocdb.log OK/ {nOK++;}
2135 /ocdb.log BAD/ {nBAD++;}
2136 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2137
2138 echo
2139 echo per run stats:
2140 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2141do
2142 dir=$(goPrintValues calibfile - ${x})
2143 runNumber=$(guessRunNumber ${dir})
2144 [[ -z ${runNumber} ]] && continue
2145
2146 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2147 statusCPass0=( $(
2148 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2149 /\/rec.log OK/ {nOKrec++;}
2150 /\/rec.log BAD/ {nBADrec++;}
2151 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2152 /stderr OK/ {nOKstderr++;}
2153 /\/calib.log OK/ {nOKcalib++;}
2154 /\/calib.log BAD/ {nBADcalib++}
2155 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2156 ) )
2157 fi
2158
2159 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2160 statusCPass1=( $(
2161 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2162 /\/rec.log OK/ {nOKrec++;}
2163 /\/rec.log BAD/ {nBADrec++;}
2164 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2165 /stderr OK/ {nOKstderr++;}
2166 /\/calib.log OK/ {nOKcalib++;}
2167 /\/calib.log BAD/ {nBADcalib++}
2168 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2169 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2170 /\/qa_outer.log OK/ {nQAouterOK++;}
2171 /\/qa_outer.log BAD/ {nQAouterBAD++;}
2172 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2173 ) )
2174 fi
2175
2176 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2177 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2178 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2179
2180 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2181done
2182
2183 #make lists with output files - QA, trending, filtering and calibration
2184 ### wait for the merging of all runs to be over ###
2185 rm -f qa.list
2186 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2187 rm -f calib.list
2188 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2189 rm -f trending.list
2190 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2191 rm -f filtering.list
444b9916 2192 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2193 rm -f cpass0.dcsTree.list
2194 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2195 rm -f cpass1.dcsTree.list
2196 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2197
2198 #merge trending
2199 rm -f ${commonOutputPath}/trending_merged.root
444b9916 2200 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2201
2202 goMakeSummaryTree ${commonOutputPath} 0
2203 goMakeSummaryTree ${commonOutputPath} 1
2204
839b07ca 2205 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2206
2207 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2208 goMakeMergedSummaryTree
2209
2210 #if set, email the summary
5de3064b 2211 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
71353a5f 2212
2213 return 0
78d0bbfd 2214)
71353a5f 2215
2216goMakeSummaryTree()
78d0bbfd 2217(
71353a5f 2218 if [[ $# -lt 1 ]] ; then
2219 return
2220 fi
2221 #1. define vars/arrays
2222 DIR=${1} #use input or exec in current dir
2223 pass=${2-"0"} #pass from input
2224 outfile="summary_pass${pass}.tree"
2225 Ncolumns=0
2226 test -f ${outfile} && : >${outfile}
2227 errfile=${outfile/tree/err}
2228 test -f ${errfile} && : >${errfile}
2229
2230 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2231 Ncounter=${#counterString[@]}
2232
2233 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2234 Nstatus=${#statusString[@]}
2235
2236
2237 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2238 Nrates=${#ratesString[@]}
2239
2240 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2241 Nruns=${#runs[@]}
2242
2243 echo -n runnumber/I >>${outfile}
2244 echo -n :cpass${pass}status/I >>${outfile}
2245 echo -n :cpass${pass}QAstatus/I >>${outfile}
2246 for i in ${ratesString[@]}; do
2247 echo -n :${i}OK/I >>${outfile}
2248 echo -n :${i}BAD/I >>${outfile}
2249
2250 done
2251 for i in ${counterString[@]} ${statusString[@]} ; do
2252 echo -n :${i}/I >>${outfile}
2253 done
2254 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2255 echo >> ${outfile}
2256
2257 #2. loop runs
2258
2259 for runnumber in ${runs[@]} ; do
2260
2261
2262
2263 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2264 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2265 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2266 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2267 echo "${filemerge} does not exist!" >>${errfile}
2268 continue
2269 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2270 echo "${filejob} does not exist!" >>${errfile}
2271 echo -n ${runnumber} >> ${outfile}
2272 for i in $(seq ${Ncolumns}) ; do
2273 echo -n "-1" >> ${outfile}
2274 done
2275 echo >> ${outfile}
2276 continue
2277 fi
2278 echo -n ${runnumber} >> ${outfile}
2279 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2280 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2281 echo -n " ${passStatus}" >> ${outfile}
2282 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2283 echo -n " ${qaStatus}" >> ${outfile}
2284
2285
2286 #fill OK/BAD rates
2287 for i in $(seq 0 $((${Nrates}-1))) ; do
2288 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2289 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2290
2291 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2292 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2293 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2294 fi
2295 echo -n " ${var1}" >> ${outfile}
2296 echo -n " ${var2}" >> ${outfile}
2297 done
2298
2299 if [[ -f ${fileOCDB} ]] ; then
2300 #fill counter
2301 for i in $(seq 0 $((${Ncounter}-1))) ; do
2302 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2303 echo -n " ${var1:-"-1"}" >> ${outfile}
2304 done
2305
2306 #fill status
2307 for i in $(seq 0 $((${Nstatus}-1))) ; do
2308 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2309 echo -n " ${var1:-"-1"}" >> ${outfile}
2310 done
2311 fi
2312 echo >> ${outfile}
2313 done
2314
5de3064b 2315 return 0
78d0bbfd 2316)
71353a5f 2317
2318parseConfig()
2319{
2320 configFile=${1}
2321 shift
444b9916 2322 args=("$@")
2323
0eaf38c7 2324
444b9916 2325 #some defaults
6fc48f9e 2326 #autoOCDB=0
2327 defaultOCDB="raw://"
2328 #runNumber=167123
2329 #makeflowPath="/hera/alice/aux/cctools/bin"
2330 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2331 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2332 makeflowOptions=""
39527a15 2333 #batchCommand="/usr/bin/qsub"
6fc48f9e 2334 batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2335 baseOutputDirectory="$PWD/output"
2336 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2337 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2338 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2339 reconstructInTemporaryDir=0
2340 recoTriggerOptions="\"\""
2341 percentProcessedFilesToContinue=100
2342 maxSecondsToWait=$(( 3600*24 ))
2343 nEvents=-1
2344 nMaxChunks=0
2345 postSetUpActionCPass0=""
2346 postSetUpActionCPass1=""
2347 runCPass0reco=1
2348 runCPass0MergeMakeOCDB=1
2349 runCPass1reco=1
2350 runCPass1MergeMakeOCDB=1
2351 runESDfiltering=1
444b9916 2352 filteringFactorHighPt=1e2
2353 filteringFactorV0s=1e1
6fc48f9e 2354 MAILTO=""
2355 #pretend=1
2356 #dontRedirectStdOutToLog=1
2357 logToFinalDestination=1
2358 ALIROOT_FORCE_COREDUMP=1
0eaf38c7 2359 pretendDelay=0
71353a5f 2360
2361 #first, source the config file
2362 if [ -f ${configFile} ]; then
2363 source ${configFile}
2364 else
0eaf38c7 2365 echo "config file ${configFile} not found!"
2366 return 1
71353a5f 2367 fi
2368
839b07ca 2369 unset encodedSpaces
2370 for opt in "${args[@]}"; do
0eaf38c7 2371 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
839b07ca 2372 done
2373
d56eeaab 2374 #then, parse the options as they override the options from file
2375 for opt in "${args[@]}"; do
0eaf38c7 2376 [[ -z ${opt} ]] && continue
2377 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2378 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
d56eeaab 2379 if [[ ! "${opt}" =~ .*=.* ]]; then
0eaf38c7 2380 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
d56eeaab 2381 return 1
2382 fi
2383 local var="${opt%%=*}"
2384 local value="${opt#*=}"
2385 echo "${var} = ${value}"
444b9916 2386 export ${var}="${value}"
71353a5f 2387 done
2388
6fc48f9e 2389 #do some checking
2390 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2391
71353a5f 2392 #export the aliroot function if defined to override normal behaviour
2393 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
6fc48f9e 2394
5de3064b 2395 return 0
71353a5f 2396}
2397
2398aliroot()
2399{
2400 args="$@"
6fc48f9e 2401 if [[ -n ${useProfilingCommand} ]]; then
71353a5f 2402 valgrindLogFile="cpu.txt"
2403 [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2404 [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
6fc48f9e 2405 [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2406 echo running ${useProfilingCommand} aliroot ${args}
2407 ${useProfilingCommand} aliroot ${args}
71353a5f 2408 else
2409 #to prevent an infinite recursion use "command aliroot" to disable
2410 #aliases and functions
2411 echo running command aliroot ${args}
444b9916 2412 command aliroot "$@"
71353a5f 2413 fi
5de3064b 2414 return 0
71353a5f 2415}
2416
2417guessRunData()
2418{
2419 #guess the period from the path, pick the rightmost one
2420 period=""
2421 runNumber=""
2422 year=""
2423 pass=""
2424 legoTrainRunNumber=""
2425 dataType=""
2426
2427 local shortRunNumber=""
2428 local IFS="/"
2429 declare -a path=( $1 )
2430 local dirDepth=$(( ${#path[*]}-1 ))
2431 i=0
2432 #for ((x=${dirDepth};x>=0;x--)); do
2433 for ((x=0;x<=${dirDepth};x++)); do
2434
2435 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2436 local field=${path[${x}]}
2437 local fieldNext=${path[$((x+1))]}
2438
2439 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2440 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2441 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2442 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2443 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2444 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2445 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2446 (( i++ ))
2447 done
2448 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2449 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2450
2451 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2452 if [[ -z ${runNumber}} ]];
2453 then
2454 #error condition
2455 return 1
2456 else
2457 #ALL OK
2458 return 0
2459 fi
5de3064b 2460 return 0
71353a5f 2461}
2462
839b07ca 2463#these functions encode strings to and from a space-less form
2464#use when spaces are not well handled (e.g. in arguments to
2465#commands in makeflow files, etc.
0eaf38c7 2466encSpaces()(echo "${1// /@@@@}")
2467decSpaces()(echo "${1//@@@@/ }")
839b07ca 2468
fc7e1e45 2469main "$@"