]> git.uio.no Git - u/mrichter/AliRoot.git/blame - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
some more updates
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
fc7e1e45 4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
71353a5f 10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
fc7e1e45 24 umask 0002
25 shift
71353a5f 26 case ${runMode} in
fc7e1e45 27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
71353a5f 29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 33 "MakeSummary") goMakeSummary "$@";;
cb473870 34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
fc7e1e45 36 "test") goTest "$@";;
71353a5f 37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
cb473870 39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 41 "Merge") goMerge "$@";;
42 *)
444b9916 43 ${runMode} "$@"
fc7e1e45 44 ;;
45 esac
39527a15 46 return 0
fc7e1e45 47}
48
71353a5f 49generateMC()
50{
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
54 runNumber=${1}
55 OCDBpath=${2}
56 nEventsim=${3}
57 if [[ -n ${pretend} ]]; then
0eaf38c7 58 sleep ${pretendDelay}
71353a5f 59 touch galice.root
60 else
61 if [[ -f sim.C && -f Config.C ]] ; then
62 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63 mv syswatch.log simwatch.log
64 fi
65 fi
66}
67
fc7e1e45 68goCPass0()
78d0bbfd 69(
fc7e1e45 70 umask 0002
71
71353a5f 72 targetDirectory=${1}
73 inputList=${2}
74 nEvents=${3}
75 ocdbPath=${4}
76 configFile=${5}
77 runNumber=${6}
78 jobindex=${7}
79 shift 7
6fc48f9e 80 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 81
39527a15 82 #record the working directory provided by the batch system
83 batchWorkingDirectory=${PWD}
84
71353a5f 85 #use the jobindex only if set and non-negative
86 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 89 if [[ -z ${jobindex} ]]; then
90 echo "no jobindex!"
91 return 1
92 fi
71353a5f 93 fi
94
95 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
96 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
999a843d 97 [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
71353a5f 98
99 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
100
101 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
102 ulimit -c unlimited
103 export ALIROOT_FORCE_COREDUMP
104 fi
fc7e1e45 105
cb473870 106 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
107 #at the QA stage the pass number is guessed from the path stored here.
108 #The Format is:
109 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
110 export PRODUCTION_METADATA="OutputDir=cpass0"
111
5de3064b 112 if [[ "${inputList}" =~ \.root$ ]]; then
113 infile=${inputList}
114 else
115 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
116 fi
71353a5f 117 chunkName=${infile##*/}
71353a5f 118
119 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
120 mkdir -p ${outputDir}
1795556a 121 if [[ ! -d ${outputDir} ]]; then
122 touch ${doneFile}
123 echo "cannot make ${outputDir}" >> ${doneFile}
124 return 1
125 fi
71353a5f 126
127 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
128 runpath=${outputDir}
0eaf38c7 129 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
130 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
71353a5f 131 mkdir -p ${runpath}
1795556a 132 if [[ ! -d ${runpath} ]]; then
133 touch ${doneFile}
134 echo "cannot make runpath ${runpath}" >> ${doneFile}
135 return 1
136 fi
0eaf38c7 137 if ! cd ${runpath}; then
0eaf38c7 138 touch ${doneFile}
1795556a 139 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
0eaf38c7 140 return 1
141 fi
fc7e1e45 142
5de3064b 143 #runCPassX/C expects the raw chunk to be linked in the run dir
144 #despite it being accessed by the full path
145 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 146
71353a5f 147 #####MC
148 if [[ -n ${generateMC} ]]; then
149 olddir=${PWD}
150 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
151 simrunpath=${outputDirMC}
0eaf38c7 152 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
153 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
71353a5f 154 mkdir -p ${outputDirMC}
155 mkdir -p ${simrunpath}
0eaf38c7 156 if cd ${simrunpath}; then
fc7e1e45 157
0eaf38c7 158 filesMC=(
159 "${batchWorkingDirectory}/sim.C"
160 "${batchWorkingDirectory}/rec.C"
161 "${batchWorkingDirectory}/Config.C"
162 "${batchWorkingDirectory}/OCDB_*.root"
163 )
164 for file in ${filesMC[*]}; do
165 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
166 done
fc7e1e45 167
0eaf38c7 168 generateMC ${runNumber} ${ocdbPath} ${nEvents}
169
170 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
171 cd ${olddir}
172
173 ln -s ${outputDirMC}/* ${runpath}/
174
175 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
176 infile=""
177 fi
71353a5f 178 fi
179 ######
180
1795556a 181 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
182 touch ${doneFile}
183 echo "input file ${inputList} not found, exiting..." >> ${doneFile}
184 return 1
185 fi
71353a5f 186
187 logOutputDir=${runpath}
188 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 189 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
190 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 191 echo "${0} $*"
fc7e1e45 192
193 echo "#####################"
194 echo CPass0:
195 echo JOB setup
71353a5f 196 echo nEvents ${nEvents}
197 echo runNumber ${runNumber}
198 echo ocdbPath ${ocdbPath}
199 echo infile ${infile}
200 echo chunkName ${chunkName}
201 echo jobindex ${jobindex}
202 echo recoTriggerOptions ${recoTriggerOptions}
203 echo targetDirectory ${targetDirectory}
204 echo commonOutputPath ${commonOutputPath}
205 echo doneFile ${doneFile}
1795556a 206 echo batchWorkingDirectory=$batchWorkingDirectory
71353a5f 207 echo runpath ${runpath}
208 echo outputDir ${outputDir}
71353a5f 209 echo PWD ${PWD}
333bc229 210 echo ALICE_ROOT ${ALICE_ROOT}
fc7e1e45 211 echo "########## ###########"
212
71353a5f 213 alirootInfo > ALICE_ROOT.log
fc7e1e45 214
215 filesCPass0=(
39527a15 216 "${batchWorkingDirectory}/runCPass0.sh"
217 "${batchWorkingDirectory}/recCPass0.C"
218 "${batchWorkingDirectory}/runCalibTrain.C"
219 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
220 "${batchWorkingDirectory}/OCDB.root"
71353a5f 221 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
222 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
223 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 224 )
225
226 for file in ${filesCPass0[*]}; do
227 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 228 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 229 done
230
71353a5f 231 echo "this directory (${PWD}) contents:"
232 /bin/ls
fc7e1e45 233 echo
234 chmod u+x runCPass0.sh
235
71353a5f 236 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
237
238 if [[ -n ${postSetUpActionCPass0} ]]; then
239 echo "running ${postSetUpActionCPass0}"
240 eval ${postSetUpActionCPass0}
fc7e1e45 241 fi
242
243 #run CPass0
71353a5f 244 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
245 if [[ -n ${pretend} ]]; then
0eaf38c7 246 sleep ${pretendDelay}
cb473870 247 touch AliESDs.root
248 touch AliESDfriends.root
fc7e1e45 249 touch AliESDfriends_v1.root
250 touch rec.log
251 touch calib.log
252 else
71353a5f 253 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
254 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 255 fi
256
fc7e1e45 257 #move stuff to final destination
71353a5f 258 echo "this directory (${PWD}) contents:"
259 /bin/ls
fc7e1e45 260 echo
261
71353a5f 262 echo rm -f ./${chunkName}
263 rm -f ./${chunkName}
0eaf38c7 264 echo "cp -R ${runpath}/* ${outputDir}"
265 cp -p -R ${runpath}/* ${outputDir}
fc7e1e45 266 echo
5ed7c82c 267
268 #validate CPass0
269 cd ${outputDir}
71353a5f 270 touch ${doneFile}
71353a5f 271 echo "dir ${outputDir}" >> ${doneFile}
ab946185 272 if summarizeLogs >> ${doneFile}; then
273 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
274 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
275 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
276 fi
fc7e1e45 277
0eaf38c7 278 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
5de3064b 279 return 0
78d0bbfd 280)
fc7e1e45 281
282goCPass1()
78d0bbfd 283(
fc7e1e45 284 umask 0002
285
71353a5f 286 targetDirectory=${1}
287 inputList=${2}
288 nEvents=${3}
289 ocdbPath=${4}
290 configFile=${5}
291 runNumber=${6}
292 jobindex=${7}
293 shift 7
444b9916 294 extraOpts=("$@")
6fc48f9e 295 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 296
39527a15 297 #record the working directory provided by the batch system
298 batchWorkingDirectory=${PWD}
299
71353a5f 300 #use the jobindex only if set and non-negative
301 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
302 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
303 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 304 if [[ -z ${jobindex} ]]; then
305 echo "no jobindex!"
306 return 1
307 fi
71353a5f 308 fi
309
310 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
311 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
999a843d 312 [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass1.job${jobindex}.run${runNumber}.done"
71353a5f 313
314 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
315
316 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
317 ulimit -c unlimited
318 export ALIROOT_FORCE_COREDUMP
319 fi
fc7e1e45 320
cb473870 321 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
322 #at the QA stage the pass number is guessed from the path stored here.
323 #The Format is:
324 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
325 export PRODUCTION_METADATA="OutputDir=cpass1"
326
1795556a 327 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
328 touch ${doneFile}
329 echo "input file ${inputList} not found, exiting..." >> ${doneFile}
330 return 1
331 fi
fc7e1e45 332 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 333 infile=${inputList}
fc7e1e45 334 else
71353a5f 335 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 336 fi
fc7e1e45 337 chunkName=${infile##*/}
5de3064b 338
71353a5f 339 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
340 mkdir -p ${outputDir}
1795556a 341 if [[ ! -d ${outputDir} ]];then
342 touch ${doneFile}
343 echo "cannot make ${outputDir}" >> ${doneFile}
344 return 1
345 fi
fc7e1e45 346
71353a5f 347 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
348 runpath=${outputDir}
0eaf38c7 349 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
350 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
71353a5f 351
352 #MC
353 if [[ "${infile}" =~ galice\.root ]]; then
354 ln -s ${inputList%/*}/* ${runpath}
355 infile=""
356 fi
fc7e1e45 357
358 #init the running path
71353a5f 359 mkdir -p ${runpath}
1795556a 360 if [[ ! -d ${runpath} ]]; then
361 touch ${doneFile}
362 echo "cannot make runpath ${runpath}" >> ${doneFile}
363 return 1
364 fi
0eaf38c7 365 if ! cd ${runpath}; then
0eaf38c7 366 touch ${doneFile}
1795556a 367 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
0eaf38c7 368 return 1
369 fi
fc7e1e45 370
71353a5f 371 #this is needed for runCPass1.sh
372 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 373
71353a5f 374 logOutputDir=${runpath}
375 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 376 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
377 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 378 echo "${0} $*"
fc7e1e45 379
380 echo "#####################"
381 echo CPass1:
382 echo JOB setup
71353a5f 383 echo nEvents ${nEvents}
384 echo runNumber ${runNumber}
385 echo ocdbPath ${ocdbPath}
386 echo infile ${infile}
387 echo chunkName ${chunkName}
388 echo jobindex ${jobindex}
389 echo recoTriggerOptions ${recoTriggerOptions}
390 echo targetDirectory ${targetDirectory}
391 echo commonOutputPath ${commonOutputPath}
392 echo doneFile ${doneFile}
393 echo runpath ${runpath}
394 echo outputDir ${outputDir}
1795556a 395 echo batchWorkingDirectory ${batchWorkingDirectory}
333bc229 396 echo ALICE_ROOT ${ALICE_ROOT}
1795556a 397 echo PWD ${PWD}
fc7e1e45 398 echo "########## ###########"
399
71353a5f 400 alirootInfo > ALICE_ROOT.log
fc7e1e45 401
402 filesCPass1=(
39527a15 403 "${batchWorkingDirectory}/runCPass1.sh"
404 "${batchWorkingDirectory}/recCPass1.C"
405 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
406 "${batchWorkingDirectory}/runCalibTrain.C"
407 "${batchWorkingDirectory}/QAtrain_duo.C"
408 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
0eaf38c7 409 "${batchWorkingDirectory}/${configFile}"
617f8886 410 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 411 "${batchWorkingDirectory}/OCDB.root"
71353a5f 412 "${trustedQAtrainMacro}"
413 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
414 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
415 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
416 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
417 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 418 )
419
78d0bbfd 420 for file in "${filesCPass1[@]}"; do
fc7e1e45 421 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
78d0bbfd 422 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
fc7e1e45 423 done
424
71353a5f 425 echo "this directory (${PWD}) contents:"
426 /bin/ls
fc7e1e45 427 echo
428
78d0bbfd 429 #remove spaces around commas from calls to root
71353a5f 430 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
431
432 if [[ -n ${postSetUpActionCPass1} ]]; then
433 echo "running ${postSetUpActionCPass1}"
434 eval ${postSetUpActionCPass1}
fc7e1e45 435 echo
436 fi
437
438 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
439 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 440 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
441 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 442 else
443 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
444 fi
445
71353a5f 446 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
71353a5f 447 touch ${doneFile}
1795556a 448 echo "cpass0 produced no calibration! exiting..." >> ${doneFile}
71353a5f 449 return 1
cb473870 450 fi
fc7e1e45 451
71353a5f 452 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
453 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
454 #entries using a relative path, e.g. local://./OCDB
78d0bbfd 455 echo "linking the OCDB/ for Barrel and OuterDet directories"
71353a5f 456 mkdir Barrel OuterDet
78d0bbfd 457 ls -l
71353a5f 458 ln -s ../OCDB Barrel/OCDB
459 ln -s ../OCDB OuterDet/OCDB
460
444b9916 461 #setup the filtering
462 #the following option enables the filtering task inside the QAtrain_duo.C
463 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
464 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
465 if [[ -n ${filteringFactorHighPt} ]]; then
466 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
467 fi
468 if [[ -n ${filteringFactorV0s} ]]; then
469 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
470 fi
471
fc7e1e45 472 #run CPass1
473 chmod u+x runCPass1.sh
71353a5f 474 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
475 if [[ -n ${pretend} ]]; then
0eaf38c7 476 sleep ${pretendDelay}
cb473870 477 touch AliESDs_Barrel.root
478 touch AliESDfriends_Barrel.root
fc7e1e45 479 touch AliESDfriends_v1.root
cb473870 480 touch QAresults_barrel.root
481 touch EventStat_temp_barrel.root
482 touch AODtpITS.root
483 touch AliESDs_Outer.root
484 touch AliESDfriends_Outer.root
485 touch QAresults_outer.root
486 touch EventStat_temp_outer.root
fc7e1e45 487 touch rec.log
488 touch calib.log
489 touch qa.log
444b9916 490 touch filtering.log FilterEvents_Trees.root
fc7e1e45 491 else
71353a5f 492 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 493
494 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
495 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
496 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
497 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
498 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
499 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
500 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
501 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
502
503 #make the filtered tree (if requested and not already produced by QA
504 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
fab31805 505 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
506 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
507 else
508 echo ""
444b9916 509 fi
510
fc7e1e45 511 fi
cb473870 512
513 ##handle possible crashes in QA (happens often in trunk)
514 ##rerun QA with a trusted aliroot version
515 #if [[ $(validateLog qa_barrel.log) ]]; then
516 # echo "qa_barrel.log not validated!"
517 #fi
518 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 519 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 520 # source ${setupTrustedAliROOTenvInCurrentShell}
521 # cd Barrel
522 # rm QAresults_barrel.root
523 # rm EventStat_temp_barrel.root
524 # rm AODtpITS.root
525 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
526 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 527 # if [[ -n ${trustedQAtrainMacro} ]]; then
528 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 529 # fi
71353a5f 530 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
531 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 532 # cd ../
533 #fi
cb473870 534
fc7e1e45 535 #move stuff to final destination
71353a5f 536 echo "this directory (${PWD}) contents:"
537 /bin/ls
538 echo rm -f ./${chunkName}
539 rm -f ./${chunkName}
0eaf38c7 540 echo "cp -R ${runpath}/* ${outputDir}"
541 cp -pf -R ${runpath}/* ${outputDir}
fc7e1e45 542 echo
543
5ed7c82c 544 #validate CPass1
545 cd ${outputDir}
71353a5f 546 touch ${doneFile}
71353a5f 547 echo "dir ${outputDir}" >> ${doneFile}
ab946185 548 if summarizeLogs >> ${doneFile}; then
549 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
550 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
551 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
552 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
34ef8842 553 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
554 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
ab946185 555 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
922d0b10 556 else
557 if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
558 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
559 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
560 fi
561 if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
562 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
563 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
564 fi
565 if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
566 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
567 fi
ab946185 568 fi
569
71353a5f 570 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 571 return 0
78d0bbfd 572)
fc7e1e45 573
574
575goMergeCPass0()
78d0bbfd 576(
fc7e1e45 577 #
578 # find the output files and merge them
579 #
580
71353a5f 581 outputDir=${1}
582 ocdbStorage=${2}
583 configFile=${3}
584 runNumber=${4}
585 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
586 shift 5
6fc48f9e 587 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 588
39527a15 589 #record the working directory provided by the batch system
590 batchWorkingDirectory=${PWD}
591
71353a5f 592 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
593 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
594
595 umask 0002
596 ulimit -c unlimited
fc7e1e45 597
71353a5f 598 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 599
71353a5f 600 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
601 runpath=${outputDir}
0eaf38c7 602 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
603 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
fc7e1e45 604
71353a5f 605 mkdir -p ${runpath}
1795556a 606 if [[ ! -d ${runpath} ]]; then
607 touch ${doneFile}
608 echo "not able to make the runpath ${runpath}" >> ${doneFile}
609 return 1
610 fi
0eaf38c7 611 if ! cd ${runpath}; then
0eaf38c7 612 touch ${doneFile}
1795556a 613 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
0eaf38c7 614 return 1
615 fi
fc7e1e45 616
71353a5f 617 logOutputDir=${runpath}
618 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
1795556a 619 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
71353a5f 620 echo "${0} $*"
fc7e1e45 621
fc7e1e45 622 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 623
624 echo goMergeCPass0 SETUP:
71353a5f 625 echo runNumber=${runNumber}
626 echo outputDir=${outputDir}
627 echo ocdbStorage=${ocdbStorage}
628 echo calibrationFilesToMerge=${calibrationFilesToMerge}
629 echo mergingScript=${mergingScript}
630 echo commonOutputPath=${commonOutputPath}
631 echo runpath=${runpath}
fc7e1e45 632
633 # copy files in case they are not already there
634 filesMergeCPass0=(
39527a15 635 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
636 "${batchWorkingDirectory}/OCDB.root"
637 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 638 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
639 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
640 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
641 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
642 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 643 )
644 for file in ${filesMergeCPass0[*]}; do
645 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 646 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 647 done
648
71353a5f 649 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
650
651 alirootInfo > ALICE_ROOT.log
fc7e1e45 652
653 #
cb473870 654 echo "PWD"
71353a5f 655 /bin/ls
cb473870 656 echo "PWD/.."
71353a5f 657 /bin/ls ../
cb473870 658
fc7e1e45 659
660 #merge calibration
71353a5f 661 chmod u+x ${mergingScript}
fc7e1e45 662 mkdir -p ./OCDB
71353a5f 663 if [[ ! -f ${calibrationFilesToMerge} ]]; then
664 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
665 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 666 fi
667
71353a5f 668 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
669 if [[ -n ${pretend} ]]; then
0eaf38c7 670 sleep ${pretendDelay}
fc7e1e45 671 touch CalibObjects.root
672 touch ocdb.log
673 touch merge.log
617f8886 674 touch dcsTime.root
71353a5f 675 mkdir -p ./OCDB/TPC/Calib/TimeGain/
676 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
1795556a 677 echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
678 echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 679 else
1795556a 680 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
fc7e1e45 681
617f8886 682 #produce the calib trees for expert QA (dcsTime.root)
683 goMakeLocalOCDBaccessConfig ./OCDB
684 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
685 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
686 fi
687
fc7e1e45 688 ### produce the output
689 #tar the produced OCDB for reuse
71353a5f 690 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
fc7e1e45 691
71353a5f 692 /bin/ls
5ed7c82c 693
694 #copy all to output dir
0eaf38c7 695 cp -pf -R ${runpath}/* ${outputDir}
617f8886 696
71353a5f 697 if [[ -n ${generateMC} ]]; then
698 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
699 fi
700
fc7e1e45 701 #validate merging cpass0
5ed7c82c 702 cd ${outputDir}
71353a5f 703 touch ${doneFile}
71353a5f 704 echo "dir ${outputDir}" >> ${doneFile}
ab946185 705 if summarizeLogs >> ${doneFile}; then
706 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
707 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
708 fi
fc7e1e45 709
71353a5f 710 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 711 return 0
78d0bbfd 712)
fc7e1e45 713
714goMergeCPass1()
78d0bbfd 715(
fc7e1e45 716 #
717 # find the output files and merge them
718 #
719
71353a5f 720 outputDir=${1}
721 ocdbStorage=${2}
722 configFile=${3}
723 runNumber=${4}
724 calibrationFilesToMerge=${5}
725 qaFilesToMerge=${6}
444b9916 726 filteredFilesToMerge=${7}
727 shift 7
6fc48f9e 728 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 729
39527a15 730 #record the working directory provided by the batch system
731 batchWorkingDirectory=${PWD}
732
71353a5f 733 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
734 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
735
736 umask 0002
737 ulimit -c unlimited
fc7e1e45 738
739 #clean up first:
71353a5f 740 rm -f ${outputDir}/*.log
741 rm -f ${outputDir}/*.root
742 rm -f ${outputDir}/*done
743
744 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
745
746 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
747 runpath=${outputDir}
0eaf38c7 748 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
749 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
71353a5f 750
751 mkdir -p ${runpath}
1795556a 752 if [[ ! -d ${runpath} ]]; then
753 touch ${doneFile}
754 echo "not able to make the runpath ${runpath}" >> ${doneFile}
755 return 1
756 fi
0eaf38c7 757 if ! cd ${runpath}; then
0eaf38c7 758 touch ${doneFile}
1795556a 759 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
0eaf38c7 760 return 1
761 fi
71353a5f 762
763 logOutputDir=${runpath}
764 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
765 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
766 echo "${0} $*"
767
768 calibrationOutputFileName='AliESDfriends_v1.root'
769 qaOutputFileName='QAresults*.root'
fc7e1e45 770 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 771 #important to have the string "Stage.txt" in the filename to trigger the merging
772 #it has to be a list of directories containing the files
fc7e1e45 773 qaMergedOutputFileName="QAresults_merged.root"
774
775 echo goMergeCPass1 SETUP:
71353a5f 776 echo runNumber=${runNumber}
777 echo outputDir=${outputDir}
778 echo ocdbStorage=${ocdbStorage}
779 echo calibrationFilesToMerge=$calibrationFilesToMerge
780 echo qaFilesToMerge=$qaFilesToMerge
781 echo calibrationOutputFileName=${calibrationOutputFileName}
782 echo mergingScript=${mergingScript}
fc7e1e45 783
784 # copy files in case they are not already there
785 filesMergeCPass1=(
39527a15 786 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
787 "${batchWorkingDirectory}/${qaFilesToMerge}"
788 "${batchWorkingDirectory}/OCDB.root"
789 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
3fbef3b9 790 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 791 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 792 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
793 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
794 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
795 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
796 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
797 "${trustedQAtrainMacro}"
798 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 799 )
800 for file in ${filesMergeCPass1[*]}; do
801 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 802 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 803 done
804
71353a5f 805 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
806
cb473870 807 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
808 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 809 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
810 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 811 else
812 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
813 fi
814
71353a5f 815 alirootInfo > ALICE_ROOT.log
fc7e1e45 816
817 #
71353a5f 818 /bin/ls
fc7e1e45 819
820 #merge calibration
71353a5f 821 chmod u+x ${mergingScript}
fc7e1e45 822 mkdir -p OCDB
444b9916 823
824 #if not provided, create the lists of files to merge
825 if [[ ! -f ${filteredFilesToMerge} ]]; then
826 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
827 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
828 fi
71353a5f 829 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 830 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
831 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
832 fi
833 if [[ ! -f ${qaFilesToMerge} ]]; then
834 #find the files, but only store the directories (QAtrain_duo.C requires this)
835 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
836 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 837 fi
838
71353a5f 839 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
840 if [[ -n ${pretend} ]]; then
0eaf38c7 841 sleep ${pretendDelay}
fc7e1e45 842 touch ocdb.log
71353a5f 843 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 844 touch ${qaMergedOutputFileName}
fc7e1e45 845 touch merge.log
cb473870 846 touch trending.root
444b9916 847 touch FilterEvents_Trees.root
ab946185 848 touch CalibObjects.root
849 touch dcsTime.root
850 touch ${qaMergedOutputFileName}
617f8886 851 mkdir -p OCDB
fc7e1e45 852 else
617f8886 853 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
854
855 #merge QA (and filtered trees)
856 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
857 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
858
859 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
860 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 861 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
862 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
863 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 864 mv trending_barrel.root trending.root
444b9916 865
866 #merge filtered trees
867 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922d0b10 868 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
617f8886 869
870 #produce the calib trees for expert QA
871 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
922d0b10 872 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
fc7e1e45 873 fi
617f8886 874
875 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
71353a5f 876
877 /bin/ls
5ed7c82c 878
879 #copy all to output dir
0eaf38c7 880 cp -pf -R ${runpath}/* ${outputDir}
5ed7c82c 881
fc7e1e45 882 #validate merge cpass1
5ed7c82c 883 cd ${outputDir}
71353a5f 884 touch ${doneFile}
71353a5f 885 echo "dir ${outputDir}" >> ${doneFile}
ab946185 886 if summarizeLogs >> ${doneFile}; then
887 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
888 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
889 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
890 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
891 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
922d0b10 892 else
893 if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
894 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
895 fi
896 if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
897 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
898 fi
ab946185 899 fi
71353a5f 900
901 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 902 return 0
78d0bbfd 903)
cb473870 904
905goMerge()
78d0bbfd 906(
cb473870 907 #generic root merge using CPass1 merge.C script
71353a5f 908 inputList=${1}
909 outputFile=${2}
0eaf38c7 910 configFile=${3-"benchmark.config"}
71353a5f 911 shift 3
6fc48f9e 912 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 913
914 #record the working directory provided by the batch system
915 batchWorkingDirectory=${PWD}
fc7e1e45 916
71353a5f 917 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
918 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
919 umask 0002
920 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
921 rm -f ${outputFile}
922 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 923 return 0
78d0bbfd 924)
fc7e1e45 925
cb473870 926goSubmitMakeflow()
5dfac970 927{
cb473870 928 #run
71353a5f 929 productionID=${1}
930 inputFileList=${2}
931 configFile=${3}
932 shift 3
444b9916 933 extraOpts=("$@")
6fc48f9e 934 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 935
936 #record the working directory provided by the batch system
937 batchWorkingDirectory=${PWD}
fc7e1e45 938
5dfac970 939 [[ -z ${configFile} ]] && configFile="benchmark.config"
940 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 941
71353a5f 942 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 943 echo "setting the makflow path from the config: "
944 echo " export PATH=${makeflowPath}:${PATH}"
945 export PATH=${makeflowPath}:${PATH}
946 fi
947
39527a15 948 #create the common output dir and the meta dir
949 commonOutputPath=${baseOutputDirectory}/${productionID}
950 if [[ -d ${commonOutputPath} ]]; then
951 echo "output dir ${commonOutputPath} exists!"
0eaf38c7 952 #return 1
39527a15 953 else
954 mkdir -p ${commonOutputPath}
955 fi
956 mkdir -p ${commonOutputPath}/meta
839b07ca 957
0eaf38c7 958 self=${0}
959 #if which greadlink; then self=$(greadlink -f "${0}"); fi
839b07ca 960
0eaf38c7 961 #for reference copy the setup to the output dir
839b07ca 962 cp ${self} ${commonOutputPath}
963 cp ${configFile} ${commonOutputPath}
333bc229 964 cp ${inputFileList} ${commonOutputPath}
39527a15 965
cb473870 966 #submit - use makeflow if available, fall back to old stuff when makeflow not there
967 if which makeflow; then
39527a15 968 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1795556a 969 cp benchmark.makeflow ${commonOutputPath}
cb473870 970 makeflow ${makeflowOptions} benchmark.makeflow
cb473870 971 else
972 echo "no makeflow!"
973 fi
5de3064b 974 return 0
5dfac970 975}
fc7e1e45 976
5dfac970 977goGenerateMakeflow()
78d0bbfd 978(
5dfac970 979 #generate the makeflow file
ab946185 980 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 981 productionID=${1}
982 inputFileList=${2}
983 configFile=${3}
984 shift 3
444b9916 985 extraOpts=("$@")
0eaf38c7 986
987 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
988 for (( i=0;i<${#extraOpts[@]};i++ )); do
989 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
990 done
991 extraOpts+=("encodedSpaces=1")
992
39527a15 993 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
994
1a0e38e3 995 #extra safety
996 if [[ -z ${commonOutputPath} ]]; then
997 commonOutputPath=${baseOutputDirectory}/${productionID}
998 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
999 fi
bff71da0 1000
39527a15 1001 #record the working directory provided by the batch system
1002 batchWorkingDirectory=${PWD}
fc7e1e45 1003
5dfac970 1004 [[ -z ${configFile} ]] && configFile="benchmark.config"
1005 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1006
5dfac970 1007 #these files will be made a dependency - will be copied to the working dir of the jobs
1008 declare -a copyFiles
1009 inputFiles=(
1010 "OCDB.root"
1011 "localOCDBaccessConfig.C"
cb473870 1012 "QAtrain_duo.C"
1013 "runCPass1.sh"
1014 "recCPass1.C"
1015 "recCPass1_OuterDet.C"
1016 "runCalibTrain.C"
1017 "runCPass0.sh"
1018 "recCPass0.C"
5dfac970 1019 )
1020 for file in ${inputFiles[*]}; do
1021 [[ -f ${file} ]] && copyFiles+=("${file}")
1022 done
1023
1024 #create the makeflow file
cb473870 1025 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 1026 declare -A arr_cpass0_merged arr_cpass1_merged
1027 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1028 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 1029 declare -A arr_cpass0_profiled_outputs
444b9916 1030 declare -A listOfRuns
1031 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 1032 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 1033 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 1034 [[ -z ${runNumber} ]] && continue
fc7e1e45 1035 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 1036
444b9916 1037 unset arr_cpass0_outputs
1038 unset arr_cpass1_outputs
1039 declare -a arr_cpass0_outputs
1040 declare -a arr_cpass1_outputs
fc7e1e45 1041
cb473870 1042 jobindex=0
6fc48f9e 1043 inputFile=""
5dfac970 1044 while read inputFile; do
1045 currentDefaultOCDB=${defaultOCDB}
cb473870 1046 [[ -z ${autoOCDB} ]] && autoOCDB=1
1047 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1048 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 1049 fi
fc7e1e45 1050
5dfac970 1051 #CPass0
39527a15 1052 arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
444b9916 1053 echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1054 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
5dfac970 1055
1056 #CPass1
39527a15 1057 arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1058 echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
0eaf38c7 1059 echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
5dfac970 1060 ((jobindex++))
fc7e1e45 1061
71353a5f 1062 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 1063
5dfac970 1064 #CPass0 list of Calib files to merge
39527a15 1065 arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
444b9916 1066 echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1067 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
71353a5f 1068 echo
1069
1070 #CPass0 merging
39527a15 1071 arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1072 echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
0eaf38c7 1073 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
fc7e1e45 1074
ab946185 1075 #CPass1 list of Calib/QA/ESD/filtered files
1076 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1077 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 1078 # this corresponds to the last merging stage)
39527a15 1079 arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
444b9916 1080 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1081 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
5dfac970 1082 echo
39527a15 1083 arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
ab946185 1084 echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1085 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1086 echo
39527a15 1087 arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
444b9916 1088 echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1089 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1090 echo
39527a15 1091 arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
444b9916 1092 echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1093 echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1094 echo
444b9916 1095
71353a5f 1096 #CPass1 merging
39527a15 1097 arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1098 echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
0eaf38c7 1099 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
fc7e1e45 1100
6fc48f9e 1101 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1102 if [[ -n ${profilingCommand} ]]; then
0eaf38c7 1103 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1104 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1105 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1106 jobindex="profiling"
1107
e48b4924 1108 arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
6fc48f9e 1109 echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1110 profilingCommand=$(encSpaces "${profilingCommand}")
1111 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
6fc48f9e 1112 fi
1113
444b9916 1114 done #runs
fc7e1e45 1115
5dfac970 1116 #Summary
1a0e38e3 1117 echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
0eaf38c7 1118 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
cb473870 1119 echo
1120
5de3064b 1121 return 0
78d0bbfd 1122)
cb473870 1123
1124goPrintValues()
78d0bbfd 1125(
cb473870 1126 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1127 if [[ $# -lt 3 ]]; then
1128 echo "goPrintValues key outputFile inputFiles"
1129 echo "if outputFile is \"-\" dont redirect to a file"
1130 return
1131 fi
1132 key=${1}
1133 outputFile=${2}
1134 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1135 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1136 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1137 return 0
78d0bbfd 1138)
fc7e1e45 1139
5dfac970 1140goCreateQAplots()
78d0bbfd 1141(
5dfac970 1142 umask 0002
71353a5f 1143 mergedQAfileList=${1}
1144 productionID=${2}
1145 outputDir=${3}
1146 configFile=${4}
1147 shift 4
e48b4924 1148 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 1149
39527a15 1150 #record the working directory provided by the batch system
1151 batchWorkingDirectory=${PWD}
1152
71353a5f 1153 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1154
71353a5f 1155 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
0eaf38c7 1156 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
71353a5f 1157 echo "${0} $*"
1158
1159 olddir=${PWD}
1160 mkdir -p ${outputDir}
1161 cd ${outputDir}
839b07ca 1162 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1163
1795556a 1164 if [[ -z ${pretend} ]]; then
1165 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1166 else
1167 touch pretendCreateQAplots.done
1168 fi
71353a5f 1169 cd ${olddir}
5de3064b 1170 return 0
78d0bbfd 1171)
fc7e1e45 1172
5dfac970 1173goTest()
78d0bbfd 1174(
1175 echo AA
1176)
fc7e1e45 1177
5dfac970 1178alirootInfo()
78d0bbfd 1179(
5dfac970 1180 umask 0002
71353a5f 1181 # save aliroot repository info
1182 [[ -z "${ALICE_ROOT}" ]] && return 1
1183
1184 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1185 echo "\${ROOTSYS}=${ROOTSYS}"
1186 echo "\${PATH}=${PATH}"
1187 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
0eaf38c7 1188 echo
71353a5f 1189
1190 pushd ${PWD}
1191 cd ${ALICE_ROOT}
71353a5f 1192
1193 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1194 git status
5dfac970 1195 echo ""
1196 echo ""
71353a5f 1197 git diff ${currentBranch}
1198 popd
5de3064b 1199 return 0
78d0bbfd 1200)
fc7e1e45 1201
5dfac970 1202setYear()
78d0bbfd 1203(
5dfac970 1204 #set the year
71353a5f 1205 # ${1} - year to be set
1206 # ${2} - where to set the year
1207 year1=$(guessYear ${1})
1208 year2=$(guessYear ${2})
1209 local path=${2}
1210 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1211 echo ${path}
5de3064b 1212 return 0
78d0bbfd 1213)
fc7e1e45 1214
5dfac970 1215guessPeriod()
78d0bbfd 1216(
5dfac970 1217 #guess the period from the path, pick the rightmost one
1218 local IFS="/"
71353a5f 1219 declare -a path=( ${1} )
5dfac970 1220 local dirDepth=${#path[*]}
1221 for ((x=${dirDepth}-1;x>=0;x--)); do
1222 local field=${path[${x}]}
71353a5f 1223 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1224 done
71353a5f 1225 echo ${period}
5de3064b 1226 return 0
78d0bbfd 1227)
fc7e1e45 1228
5dfac970 1229guessYear()
78d0bbfd 1230(
5dfac970 1231 #guess the year from the path, pick the rightmost one
1232 local IFS="/"
71353a5f 1233 declare -a path=( ${1} )
5dfac970 1234 local dirDepth=${#path[*]}
1235 for ((x=${dirDepth}-1;x>=0;x--)); do
1236 local field=${path[${x}]}
71353a5f 1237 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1238 done
71353a5f 1239 echo ${year}
5de3064b 1240 return 0
78d0bbfd 1241)
fc7e1e45 1242
5dfac970 1243guessRunNumber()
78d0bbfd 1244(
5dfac970 1245 #guess the run number from the path, pick the rightmost one
71353a5f 1246 #works for /path/foo/000123456/bar/...
1247 #and /path/foo.run123456.bar
1248 local IFS="/."
1249 declare -a path=( ${1} )
5dfac970 1250 local dirDepth=${#path[*]}
1251 for ((x=${dirDepth}-1;x>=0;x--)); do
1252 local field=${path[${x}]}
71353a5f 1253 field=${field/run/000}
617f8886 1254 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1255 done
71353a5f 1256 echo ${runNumber}
5de3064b 1257 return 0
78d0bbfd 1258)
fc7e1e45 1259
cb473870 1260validateLog()
78d0bbfd 1261(
71353a5f 1262 log=${1}
cb473870 1263 errorConditions=(
71353a5f 1264 'There was a crash'
1265 'floating'
1266 'error while loading shared libraries'
1267 'std::bad_alloc'
1268 's_err_syswatch_'
1269 'Thread [0-9]* (Thread'
1270 'AliFatal'
1271 'core dumped'
1272 '\.C.*error:.*\.h: No such file'
839b07ca 1273 'line.*Aborted'
71353a5f 1274 )
1275
1276 warningConditions=(
1277 'This is serious !'
1278 'rocVoltage out of range:'
cb473870 1279 )
1280
71353a5f 1281 local logstatus=0
1282 local errorSummary=""
1283 local warningSummary=""
1284
1285 for ((i=0; i<${#errorConditions[@]};i++)); do
1286 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1287 [[ -n ${tmp} ]] && tmp+=" : "
1288 errorSummary+=${tmp}
1289 done
1290
1291 for ((i=0; i<${#warningConditions[@]};i++)); do
1292 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1293 [[ -n ${tmp} ]] && tmp+=" : "
1294 warningSummary+=${tmp}
1295 done
1296
1297 if [[ -n ${errorSummary} ]]; then
1298 echo "${errorSummary}"
1299 return 1
1300 fi
cb473870 1301
71353a5f 1302 if [[ -n ${warningSummary} ]]; then
1303 echo "${warningSummary}"
1304 return 2
1305 fi
1306
1307 return 0
78d0bbfd 1308)
cb473870 1309
5dfac970 1310summarizeLogs()
78d0bbfd 1311(
5dfac970 1312 #print a summary of logs
1313 logFiles=(
1314 "*.log"
cb473870 1315 "stdout"
1316 "stderr"
5dfac970 1317 )
fc7e1e45 1318
cb473870 1319 #check logs
71353a5f 1320 local logstatus=0
5dfac970 1321 for log in ${logFiles[*]}; do
71353a5f 1322 finallog=${PWD%/}/${log}
1323 [[ ! -f ${log} ]] && continue
1324 errorSummary=$(validateLog ${log})
1325 validationStatus=$?
1326 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1327 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1328 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1329 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1330 echo "${finallog} OK"
fc7e1e45 1331 else
71353a5f 1332 echo "${finallog} OK"
fc7e1e45 1333 fi
71353a5f 1334 elif [[ ${validationStatus} -eq 1 ]]; then
1335 echo "${finallog} BAD ${errorSummary}"
ab946185 1336 logstatus=1
71353a5f 1337 elif [[ ${validationStatus} -eq 2 ]]; then
1338 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1339 fi
1340 done
cb473870 1341
1342 #report core files
71353a5f 1343 while read x; do
1344 echo ${x}
1345 chmod 644 ${x}
1346 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1347 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1348
71353a5f 1349 return ${logstatus}
78d0bbfd 1350)
fc7e1e45 1351
5dfac970 1352spitOutLocalOCDBaccessConfig()
1353{
1354 umask 0002
71353a5f 1355 #find ${1} -name "*root" | \
1356 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1357 while read line
1358 do
71353a5f 1359 local tmp=${line#${1}}
5dfac970 1360 echo ${tmp%/*} | \
71353a5f 1361 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1362 done
5de3064b 1363 return 0
5dfac970 1364}
fc7e1e45 1365
71353a5f 1366goMakeLocalOCDBaccessConfig()
5dfac970 1367{
1368 umask 0002
1369 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1370 local localOCDBpathCPass0=${1}
1371 local OCDBpathPrefix=${2}
1372 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1373
1374 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1375 tar xzf ${localOCDBpathCPass0}
1376 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1377 fi
fc7e1e45 1378
5dfac970 1379 echo
1380 echo creating the specific storage script
1381 echo localOCDBaccessConfig.C
71353a5f 1382 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1383 echo
fc7e1e45 1384
5dfac970 1385 local tempLocalOCDB=""
1386 if [[ -f localOCDBaccessConfig.C ]]; then
0eaf38c7 1387 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
71353a5f 1388 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1389 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1390 fi
fc7e1e45 1391
5dfac970 1392 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1393 echo "{" >> localOCDBaccessConfig.C
1394 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1395 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1396 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1397 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1398
71353a5f 1399 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1400
5dfac970 1401 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1402 echo
1403 echo "!!!!!!! CPass0 produced no OCDB entries"
1404 return 1
1405 fi
5de3064b 1406 return 0
5dfac970 1407}
fc7e1e45 1408
cb473870 1409goMakeFilteredTrees()
78d0bbfd 1410(
71353a5f 1411 outputDir=${1}
1412 runNumber=${2}
cb473870 1413 #get path to input list
71353a5f 1414 inputListfiles=${3}
cb473870 1415 #get scale number for tracks
71353a5f 1416 filterT=${4}
cb473870 1417 #get scale number for V0s
71353a5f 1418 filterV=${5}
cb473870 1419 #get OCDB path (optional)
1420 OCDBpath=${6}
1421 #get max number of files
1422 maxFiles=${7-"1000000"}
1423 #get offset of first file
1424 offsetFile=${8-"0"}
1425 #get max number of events
1426 maxEvents=${9-"30000000"}
1427 #get offset of first event
1428 offsetEvent=${10-"0"}
1429 configFile=${11-"benchmark.config"}
1430 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1431 shift 12
6fc48f9e 1432 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1433
1434 #record the working directory provided by the batch system
1435 batchWorkingDirectory=${PWD}
1436
78d0bbfd 1437 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
71353a5f 1438 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1439
cb473870 1440 cat > filtering.log << EOF
1441 goMakeFilteredTrees config:
71353a5f 1442 runpath=${runpath}
1443 outputDir=${outputDir}
1444 commonOutputPath=${commonOutputPath}
1445 ALICE_ROOT=${ALICE_ROOT}
1446 PATH=${PATH}
1447 offsetEvent=$offsetEvent
1448 configFile=$configFile
1449 esdFileName=$esdFileName
fab31805 1450 inputListfiles=$inputListfiles
1451 doneFile=$doneFile
cb473870 1452EOF
1453
fab31805 1454 #runpath=${outputDir}
1455 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1456 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1457 #mkdir -p ${outputDir}
1458 #mkdir -p ${runpath}
1459 #if ! cd ${runpath}; then
1460 # echo "PWD=$PWD is not the runpath=${runpath}"
1461 # touch ${doneFile}
1462 # return 1
1463 #fi
1464
cb473870 1465 if [[ -z ${pretend} ]];then
71353a5f 1466 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1467 else
0eaf38c7 1468 sleep ${pretendDelay}
71353a5f 1469 touch filtering.log FilterEvents_Trees.root
cb473870 1470 fi
1471 pwd
71353a5f 1472 /bin/ls
1473 touch ${doneFile}
1474 summarizeLogs >> ${doneFile}
fab31805 1475
1476 #echo mv -f * ${outputDir}
1477 #mv -f * ${outputDir}
1478 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1479 #cd ${commonOutputPath}
1480 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1481
5de3064b 1482 return 0
78d0bbfd 1483)
cb473870 1484
1485submit()
1486{
1487 umask 0002
444b9916 1488 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1489 JobID=${1}
1490 startID=${2}
1491 endID=${3}
1492 waitForJOBID=${4}
1493 command=${5}
444b9916 1494 shift 5
78d0bbfd 1495 local commandArgs=("$@")
444b9916 1496
1497 #add quote strings around the extra arguments
1498 for ((i=0; i<${#commandArgs[@]}; i++)); do
1499 commandArgs[i]=\"${commandArgs[i]}\"
1500 done
cb473870 1501
71353a5f 1502 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1503
1504 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1505
1506 batchSystem="SGE"
1507
71353a5f 1508 if [[ -z "${newFarm}" ]]
cb473870 1509 then
1510 #old LSF
1511 # submit it (as job array)
71353a5f 1512 nFiles=$(( ${endID}-${startID}+1 ))
1513 while [ ${startID} -le ${nFiles} ] ; do
1514 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1515 endID=$(expr ${startID} + 999)
cb473870 1516 else
71353a5f 1517 endID=${nFiles}
cb473870 1518 fi
1519 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1520 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1521 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1522 else
71353a5f 1523 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1524 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1525 fi
71353a5f 1526 startID=$(expr ${endID} + 1)
cb473870 1527 done
1528 else
1529 #new SGE farm
1530 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1531 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1532 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1533 else
444b9916 1534 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1535 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1536 fi
1537 fi
5de3064b 1538 return 0
cb473870 1539}
1540
1541goSubmitBatch()
5dfac970 1542{
71353a5f 1543 if [[ $# -lt 3 ]]; then
1544 echo "minimal use:"
1545 echo " ${0} submit fileList productionID configFile"
1546 return 0
1547 fi
1548
1549 productionID=${1}
1550 inputList=${2}
1551 configFile=${3:-"benchmark.config"}
0eaf38c7 1552 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
71353a5f 1553 shift 3
444b9916 1554 extraOpts=("$@")
6fc48f9e 1555 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1556
b0e401b4 1557 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1558 for (( i=0;i<${#extraOpts[@]};i++ )); do
1559 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1560 done
1561 extraOpts+=("encodedSpaces=1")
1562
39527a15 1563 #record the working directory provided by the batch system
1564 batchWorkingDirectory=${PWD}
fc7e1e45 1565
5dfac970 1566 #redirect all output to submit.log
1567 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1568 exec 7>&1
1569 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1570
5dfac970 1571 umask 0002
71353a5f 1572 echo ${0}" submit $*"
1573 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1574 then
fc7e1e45 1575 echo
71353a5f 1576 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1577 echo
71353a5f 1578 return
fc7e1e45 1579 fi
fc7e1e45 1580
5dfac970 1581 # check if config file is there
71353a5f 1582 if [ ! -f ${configFile} ]; then
1583 echo "ERROR! Config File '${configFile}' not found" >&2
1584 return
5dfac970 1585 else
71353a5f 1586 echo "Using Config File: '${configFile}'"
fc7e1e45 1587 fi
1588
9510565c 1589 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1590
71353a5f 1591 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
0eaf38c7 1592 self=${0}
1593 #if which greadlink; then self=$(greadlink -f "${0}"); fi
71353a5f 1594 configPath=$(dirname ${configFile})
5dfac970 1595 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1596
cb473870 1597 mkdir -p ${commonOutputPath}
71353a5f 1598 mkdir -p ${commonOutputPath}/logs
1599 mkdir -p ${commonOutputPath}/meta
1600
1601 cp ${self} ${commonOutputPath}
1602 cp ${configFile} ${commonOutputPath}
1603 cp ${inputList} ${commonOutputPath}
cb473870 1604 self=${commonOutputPath}/${self##*/}
71353a5f 1605 chmod u+x ${self}
cb473870 1606 configFile=${commonOutputPath}/${configFile##*/}
1607 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1608
5dfac970 1609 #convert to absolut pathnames
0eaf38c7 1610 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
5dfac970 1611 #make list of runs
71353a5f 1612 if [[ -z ${runNumber} ]]; then
1613 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1614 else
71353a5f 1615 listOfRuns=${runNumber}
5dfac970 1616 fi
fc7e1e45 1617
0eaf38c7 1618 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
fc7e1e45 1619
5dfac970 1620 echo ""
1621 echo "### BEGIN CONFIGURATION ###"
1622 echo ""
1623 echo "GENERAL:"
1624 echo ""
71353a5f 1625 echo " productionID: ${productionID}"
1626 echo " batchCommand: ${batchCommand}"
1627 echo " batchFlags: ${batchFlags}"
9510565c 1628 echo " alirootEnv: ${alirootEnv}"
1629 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1630 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1631 echo " inputList: ${inputList}"
1632 echo " configPath: ${configPath}"
1633 echo " commonOutputPath: ${commonOutputPath}"
1634 echo " defaultOCDB: ${defaultOCDB}"
1635 echo " autoOCDB: ${autoOCDB}"
1636 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1637 echo " runs:"
1638 echo " ${listOfRuns[*]}"
1639 echo ""
1640 echo "THE TRAIN WILL RUN:"
fc7e1e45 1641
71353a5f 1642 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1643 echo " Pass0 - Recontruction"
1644 fi
fc7e1e45 1645
71353a5f 1646 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1647 echo " Pass0 - merging and OCDB export"
1648 fi
fc7e1e45 1649
71353a5f 1650 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1651 echo " Pass1 - Recontruction"
1652 fi
71353a5f 1653 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1654 echo " Pass1 - merging and OCDB export"
1655 fi
fc7e1e45 1656
5dfac970 1657 echo ""
1658 echo "LIMITS:"
71353a5f 1659 echo " max. Events/Chunk: ${nEvents}"
1660 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1661 echo ""
1662 echo "### END CONFIGURATION ###"
1663 echo ""
fc7e1e45 1664
36b5de1a 1665
5dfac970 1666 # check if input file is there
71353a5f 1667 if [ ! -f ${inputList} ]; then
1668 echo "ERROR! Input List '${inputList}' not found" >&2
1669 return
5dfac970 1670 fi
1671
1672 # define jobid (for dependent jobs)
cb473870 1673 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1674 #for each run we submit one jobarray:
fc7e1e45 1675 for runNumber in ${listOfRuns[*]}; do
cb473870 1676
71353a5f 1677 [[ -z ${runNumber} ]] && continue
fc7e1e45 1678 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1679
1680 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1681 JOBID1="p0_${JOBpostfix}"
1682 JOBID1wait="w0_${JOBpostfix}"
1683 JOBID2="m0_${JOBpostfix}"
1684 JOBID2wait="wm0_${JOBpostfix}"
1685 JOBID3="op0_${JOBpostfix}"
1686 JOBID3wait="wop0_${JOBpostfix}"
1687 JOBID4="p1_${JOBpostfix}"
1688 JOBID4wait="w1_${JOBpostfix}"
1689 JOBID5="m1_${JOBpostfix}"
1690 JOBID5wait="wm1_${JOBpostfix}"
1691 JOBID6="s1_${JOBpostfix}"
1692 JOBID6wait="ws1_${JOBpostfix}"
1693 JOBID7="QA_${JOBpostfix}"
1694 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1695 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1696 LASTJOB="000"
1697
0eaf38c7 1698 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
cb473870 1699
1700 currentDefaultOCDB=${defaultOCDB}
1701 [[ -z ${autoOCDB} ]] && autoOCDB=1
1702 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1703 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1704 fi
1705
71353a5f 1706 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1707
1708 ###############################################################################
1709 #run one chunk with valgrind:
6fc48f9e 1710 if [[ -n ${profilingCommand} ]]; then
1711 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1712 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
e48b4924 1713 submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
71353a5f 1714 fi
fc7e1e45 1715
5dfac970 1716 ################################################################################
1717 ################################################################################
1718 # run the CPass0 if requested
fc7e1e45 1719
71353a5f 1720 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1721
fc7e1e45 1722 echo
71353a5f 1723 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1724 echo
fc7e1e45 1725
5dfac970 1726 # create directory and copy all files that are needed
cb473870 1727 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1728 mkdir -p ${targetDirectory}
1729
cb473870 1730 filesCPass0=(
71353a5f 1731 "${configPath}/runCPass0.sh"
1732 "${configPath}/recCPass0.C"
1733 "${configPath}/runCalibTrain.C"
1734 "${configPath}/localOCDBaccessConfig.C"
1735 "${configPath}/OCDB*.root"
1736 "${configPath}/sim.C"
1737 "${configPath}/rec.C"
1738 "${configPath}/Config.C"
cb473870 1739 )
1740 for file in ${filesCPass0[*]}; do
1741 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1742 done
fc7e1e45 1743
71353a5f 1744 localInputList=${targetDirectory}/${inputList##*/}
1745 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1746 # limit nFiles to nMaxChunks
71353a5f 1747 nFiles=$(wc -l < ${localInputList})
1748 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1749 echo "raw files in list: ${nFiles}"
1750 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1751 nFiles=${nMaxChunks}
5dfac970 1752 fi
71353a5f 1753 echo "raw files to process: ${nFiles}"
1754 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1755 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1756 nFilesToWaitFor=${nFiles}
5dfac970 1757 else
71353a5f 1758 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1759 fi
71353a5f 1760 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1761 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1762
444b9916 1763 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1764
5dfac970 1765 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 1766 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 1767 LASTJOB=${JOBID1wait}
fc7e1e45 1768
5dfac970 1769 fi #end running CPass0
1770 ################################################################################
fc7e1e45 1771
fc7e1e45 1772
5dfac970 1773 ################################################################################
1774 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1775
71353a5f 1776 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1777
5dfac970 1778 echo
71353a5f 1779 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1780 echo
fc7e1e45 1781
cb473870 1782 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1783 mkdir -p ${targetDirectory}
fc7e1e45 1784
cb473870 1785 #copy the scripts
1786 filesMergeCPass0=(
71353a5f 1787 "${configPath}/OCDB.root"
1788 "${configPath}/mergeMakeOCDB.byComponent.sh"
1789 "${configPath}/mergeMakeOCDB.sh"
1790 "${configPath}/localOCDBaccessConfig.C"
1791 "${configPath}/mergeByComponent.C"
1792 "${configPath}/makeOCDB.C"
1793 "${configPath}/merge.C"
cb473870 1794 )
1795 for file in ${filesMergeCPass0[*]}; do
1796 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1797 done
1798
444b9916 1799 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1800 LASTJOB=${JOBID2}
fc7e1e45 1801
71353a5f 1802 if [[ -n ${generateMC} ]]; then
444b9916 1803 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1804 LASTJOB="mrl${JOBpostfix}"
1805 fi
fc7e1e45 1806
5dfac970 1807 echo
fc7e1e45 1808 fi
5dfac970 1809 # end of merging CPass0
1810 ################################################################################
fc7e1e45 1811
5dfac970 1812 ################################################################################
1813 ################################################################################
1814 # run the CPass1 if requested
fc7e1e45 1815
71353a5f 1816 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1817
cb473870 1818 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1819 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1820
5dfac970 1821 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1822 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1823
5dfac970 1824 echo
71353a5f 1825 echo "starting CPass1... for run ${runNumber}"
5dfac970 1826 echo
1827
1828 # create directory and copy all files that are needed
71353a5f 1829 mkdir -p ${targetDirectory}
5dfac970 1830
cb473870 1831 filesCPass1=(
71353a5f 1832 "${configPath}/runCPass1.sh"
1833 "${configPath}/recCPass1.C"
1834 "${configPath}/recCPass1_OuterDet.C"
1835 "${configPath}/runCalibTrain.C"
1836 "${configPath}/QAtrain_duo.C"
1837 "${configPath}/localOCDBaccessConfig.C"
1838 "${configPath}/OCDB.root"
cb473870 1839 )
1840 for file in ${filesCPass1[*]}; do
1841 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1842 done
5dfac970 1843
71353a5f 1844 if [[ -n ${generateMC} ]]; then
1845 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1846 else
1847 localInputList=${targetDirectory}/${inputList##*/}
1848 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1849 fi
5dfac970 1850 # limit nFiles to nMaxChunks
71353a5f 1851 nFiles=$(wc -l < ${localInputList})
1852 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1853 echo "raw files in list: ${nFiles}"
1854 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1855 nFiles=${nMaxChunks}
5dfac970 1856 fi
71353a5f 1857 echo "raw files to process: ${nFiles}"
1858 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1859 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1860 nFilesToWaitFor=${nFiles}
fc7e1e45 1861 else
71353a5f 1862 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 1863 fi
71353a5f 1864 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1865 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1866
444b9916 1867 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1868
5dfac970 1869 ################################################################################
1870 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 1871 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 1872 LASTJOB=${JOBID4wait}
5dfac970 1873 ################################################################################
fc7e1e45 1874
5dfac970 1875 echo
1876 fi #end running CPass1
fc7e1e45 1877
5dfac970 1878 ################################################################################
1879 # submit merging of CPass1, depends on the reconstruction
71353a5f 1880 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1881
5dfac970 1882 echo
71353a5f 1883 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 1884 echo
fc7e1e45 1885
cb473870 1886 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1887 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1888 mkdir -p ${targetDirectory}
fc7e1e45 1889
cb473870 1890 # copy files
1891 filesMergeCPass1=(
71353a5f 1892 "${configPath}/OCDB.root"
1893 "${configPath}/localOCDBaccessConfig.C"
1894 "${configPath}/mergeMakeOCDB.byComponent.sh"
1895 "${configPath}/mergeByComponent.C"
1896 "${configPath}/makeOCDB.C"
1897 "${configPath}/merge.C"
1898 "${configPath}/mergeMakeOCDB.sh"
1899 "${configPath}/QAtrain_duo.C"
cb473870 1900 )
1901 for file in ${filesMergeCPass1[*]}; do
1902 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1903 done
1904
444b9916 1905 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1906 LASTJOB=${JOBID5}
5dfac970 1907 echo
1908 fi
1909
444b9916 1910 ###############################
1911 #if [ ${runESDfiltering} -eq 1 ]; then
1912 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1913 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1914 # echo
1915 # echo submitting filtering for run ${runNumber}
1916 # echo
1917 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1918 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1919 # LASTJOB=${JOBfilterESDcpass1}
1920 #fi
cb473870 1921
5dfac970 1922 done
1923
71353a5f 1924 #################################################################################
1925 #################################################################################
444b9916 1926 #if [ ${runESDfiltering} -eq 1 ]; then
1927 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1928 #else
1929 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1930 #fi
71353a5f 1931 LASTJOB=${JOBID5wait}
cb473870 1932
5de3064b 1933 #################################################################################
1934 echo
1935 echo "submit make a summary"
1936 echo
5dfac970 1937
444b9916 1938 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
5de3064b 1939 LASTJOB=${JOBID6}
71353a5f 1940 #################################################################################
5dfac970 1941
5dfac970 1942 #restore stdout
1943 exec 1>&7 7>&-
1944 echo "jobs submitted."
5de3064b 1945 return 0
fc7e1e45 1946}
1947
cb473870 1948goWaitForOutput()
78d0bbfd 1949(
cb473870 1950 umask 0002
71353a5f 1951 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1952 echo searchPath=${1}
1953 echo fileName=${2}
1954 echo numberOfFiles=${3}
1955 echo maxSecondsToWait=${4}
1956 searchPath=${1}
1957 fileName=${2}
1958 numberOfFiles=${3}
1959 maxSecondsToWait=${4}
0ff60de9 1960 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
71353a5f 1961 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 1962 while true; do
cb473870 1963 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 1964 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1965 [[ ${n} -ge ${numberOfFiles} ]] && break
1966 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 1967 sleep 60
1968 done
1969 echo "DONE! exiting..."
5de3064b 1970 return 0
78d0bbfd 1971)
cb473870 1972
71353a5f 1973mergeSysLogs()
78d0bbfd 1974(
71353a5f 1975 outputFile=${1}
1976 shift
1977 inputFiles="$@"
1978 i=0
1979 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1980 while read x; do
1981 runNumber=$(guessRunNumber ${x})
1982 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1983 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1984 (( i++ ))
1985 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 1986 return 0
78d0bbfd 1987)
71353a5f 1988
1989goMakeMergedSummaryTree()
78d0bbfd 1990(
71353a5f 1991 # create list of calibration entries
1992 # takes no arguments, just run it in the base output
1993 # directory with the following files in the working directory
1994 #
1995 # Calibration file lists:
1996 # cpass0.dcsTree.list, cpass1.dcsTree.list
1997 # QA trending root file:
1998 # trending.root
1999 #
2000 # Production infoo ascii files:
2001 # summary_pass0.tree
2002 # summary_pass1.tree
2003 #
617f8886 2004
71353a5f 2005 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2006 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2007 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2008 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2009 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2010
2011 #first, dump the C macro to file
2012 cat << EOF > mergeTree.C
2013 //
2014 // Merge summary information
2015 // Following files are expected to be in the working directory
2016 //
2017 // Calibration file lists:
2018 // cpass0.dcsTree.list, cpass1.dcsTree.list
2019 // QA trending root file:
2020 // trending.root
2021 //
2022 // Production infoo ascii files:
2023 // summary_pass0.tree
2024 // summary_pass1.tree
2025 //
2026 void mergeTree(){
2027 //
2028 //
2029 //
2030 // Calibration values dump
2031 //
839b07ca 2032 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 2033 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
839b07ca 2034 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 2035 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2036 //
2037 // Calibration status dump
2038 //
2039 TFile *fprod = TFile::Open("fproduction.root","recreate");
2040 TTree tree0, tree1;
839b07ca 2041 //Printf("reading summary_pass0.tree");
71353a5f 2042 tree0.ReadFile("summary_pass0.tree");
839b07ca 2043 //Printf("reading summary_pass1.tree");
71353a5f 2044 tree1.ReadFile("summary_pass1.tree");
2045 tree0.Write("CPass0");
2046 tree1.Write("CPass1");
2047 fprod->Close();
2048 //
2049 //
2050 //
2051 TString stringSetup="";
2052 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2053 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2054 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2055 //
2056 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2057 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2058 //
839b07ca 2059 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 2060 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2061 }
2062EOF
2063
2064 aliroot -b -q "mergeTree.C" > mergeTrees.log
2065 return $?
78d0bbfd 2066)
71353a5f 2067
2068goMakeSummary()
78d0bbfd 2069(
71353a5f 2070 #all the final stuff goes in here for ease of use:
2071 # summary logs
2072 # qa plot making
2073 # final file lists
444b9916 2074 #some defaults:
2075 log="summary.log"
2076 productionID="qa"
71353a5f 2077
2078 configFile=${1}
2079 shift 1
444b9916 2080 extraOpts=("$@")
6fc48f9e 2081 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 2082
0eaf38c7 2083 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
839b07ca 2084
39527a15 2085 #record the working directory provided by the batch system
2086 batchWorkingDirectory=${PWD}
2087
71353a5f 2088 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2089
71353a5f 2090 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2091
2092 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2093
39527a15 2094 #copy some useful stuff
2095 #and go to the commonOutputPath
2096 cp ${configFile} ${commonOutputPath}
2097 cd ${commonOutputPath}
2098
2099 exec &> >(tee ${log})
2100
71353a5f 2101 #summarize the global stuff
2102 echo "env script: ${alirootSource} ${alirootEnv}"
2103 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2104 echo "commonOutputPath=${commonOutputPath}"
2105
2106 #summarize the stacktraces
2107 awk '
2108 BEGIN {
2109 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2110 RS="#[0-9]*";
2111 aliroot=0;
2112 }
2113 {
2114 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2115 gsub("#","",RT);
2116 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2117 }
2118 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2119 awk '
2120 BEGIN {
2121 RS="#[0-9]*";
2122 aliroot=0;
2123 }
2124 {
2125 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2126 gsub("#","",RT);
2127 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2128 }
2129 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2130
2131 echo total numbers for the production:
2132 echo
2133 awk 'BEGIN {nFiles=0;nCore=0;}
2134 /^calibfile/ {nFiles++;}
2135 /core dumped/ {nCore++i;}
2136 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2137 awk 'BEGIN {nOK=0; nBAD=0; }
2138 /\/rec.log OK/ {nOK++;}
2139 /\/rec.log BAD/ {nBAD++;}
2140 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2141 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2142 awk 'BEGIN {nOK=0; nBAD=0; }
2143 /\/calib.log OK/ {nOK++;}
2144 /\/calib.log BAD/ {nBAD++;}
2145 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2146
2147 awk 'BEGIN {nOK=0; nBAD=0; }
2148 /merge.log OK/ {nOK++;}
2149 /merge.log BAD/ {nBAD++;}
2150 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2151 awk 'BEGIN {nOK=0; nBAD=0; }
2152 /ocdb.log OK/ {nOK++;}
2153 /ocdb.log BAD/ {nBAD++;}
2154 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2155
2156 echo
2157 awk 'BEGIN {nFiles=0;nCore=0;}
2158 /^calibfile/ {nFiles++;}
2159 /core dumped/ {nCore++;}
2160 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2161 awk 'BEGIN {nOK=0; nBAD=0; }
2162 /\/rec.log OK/ {nOK++;}
2163 /\/rec.log BAD/ {nBAD++;}
2164 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2165 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2166 awk 'BEGIN {nOK=0; nBAD=0; }
2167 /\/calib.log OK/ {nOK++;}
2168 /\/calib.log BAD/ {nBAD++;}
2169 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2170
2171 awk 'BEGIN {nOK=0; nBAD=0; }
2172 /merge.log OK/ {nOK++;}
2173 /merge.log BAD/ {nBAD++;}
2174 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2175 awk 'BEGIN {nOK=0; nBAD=0; }
2176 /ocdb.log OK/ {nOK++;}
2177 /ocdb.log BAD/ {nBAD++;}
2178 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2179
2180 echo
2181 echo per run stats:
2182 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2183do
2184 dir=$(goPrintValues calibfile - ${x})
2185 runNumber=$(guessRunNumber ${dir})
2186 [[ -z ${runNumber} ]] && continue
2187
2188 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2189 statusCPass0=( $(
2190 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2191 /\/rec.log OK/ {nOKrec++;}
2192 /\/rec.log BAD/ {nBADrec++;}
2193 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2194 /stderr OK/ {nOKstderr++;}
2195 /\/calib.log OK/ {nOKcalib++;}
2196 /\/calib.log BAD/ {nBADcalib++}
2197 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2198 ) )
2199 fi
2200
2201 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2202 statusCPass1=( $(
2203 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2204 /\/rec.log OK/ {nOKrec++;}
2205 /\/rec.log BAD/ {nBADrec++;}
2206 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2207 /stderr OK/ {nOKstderr++;}
2208 /\/calib.log OK/ {nOKcalib++;}
2209 /\/calib.log BAD/ {nBADcalib++}
2210 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2211 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2212 /\/qa_outer.log OK/ {nQAouterOK++;}
2213 /\/qa_outer.log BAD/ {nQAouterBAD++;}
2214 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2215 ) )
2216 fi
2217
2218 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2219 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2220 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2221
2222 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2223done
2224
2225 #make lists with output files - QA, trending, filtering and calibration
2226 ### wait for the merging of all runs to be over ###
2227 rm -f qa.list
2228 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2229 rm -f calib.list
2230 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2231 rm -f trending.list
2232 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2233 rm -f filtering.list
444b9916 2234 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2235 rm -f cpass0.dcsTree.list
2236 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2237 rm -f cpass1.dcsTree.list
2238 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2239
2240 #merge trending
2241 rm -f ${commonOutputPath}/trending_merged.root
444b9916 2242 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2243
2244 goMakeSummaryTree ${commonOutputPath} 0
2245 goMakeSummaryTree ${commonOutputPath} 1
2246
839b07ca 2247 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2248
2249 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2250 goMakeMergedSummaryTree
2251
2252 #if set, email the summary
5de3064b 2253 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
71353a5f 2254
2255 return 0
78d0bbfd 2256)
71353a5f 2257
2258goMakeSummaryTree()
78d0bbfd 2259(
71353a5f 2260 if [[ $# -lt 1 ]] ; then
2261 return
2262 fi
2263 #1. define vars/arrays
2264 DIR=${1} #use input or exec in current dir
2265 pass=${2-"0"} #pass from input
2266 outfile="summary_pass${pass}.tree"
2267 Ncolumns=0
2268 test -f ${outfile} && : >${outfile}
2269 errfile=${outfile/tree/err}
2270 test -f ${errfile} && : >${errfile}
2271
2272 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2273 Ncounter=${#counterString[@]}
2274
2275 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2276 Nstatus=${#statusString[@]}
2277
2278
2279 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2280 Nrates=${#ratesString[@]}
2281
2282 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2283 Nruns=${#runs[@]}
2284
2285 echo -n runnumber/I >>${outfile}
2286 echo -n :cpass${pass}status/I >>${outfile}
2287 echo -n :cpass${pass}QAstatus/I >>${outfile}
2288 for i in ${ratesString[@]}; do
2289 echo -n :${i}OK/I >>${outfile}
2290 echo -n :${i}BAD/I >>${outfile}
2291
2292 done
2293 for i in ${counterString[@]} ${statusString[@]} ; do
2294 echo -n :${i}/I >>${outfile}
2295 done
2296 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2297 echo >> ${outfile}
2298
2299 #2. loop runs
2300
2301 for runnumber in ${runs[@]} ; do
2302
2303
2304
2305 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2306 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2307 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2308 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2309 echo "${filemerge} does not exist!" >>${errfile}
2310 continue
2311 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2312 echo "${filejob} does not exist!" >>${errfile}
2313 echo -n ${runnumber} >> ${outfile}
2314 for i in $(seq ${Ncolumns}) ; do
2315 echo -n "-1" >> ${outfile}
2316 done
2317 echo >> ${outfile}
2318 continue
2319 fi
2320 echo -n ${runnumber} >> ${outfile}
2321 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2322 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2323 echo -n " ${passStatus}" >> ${outfile}
2324 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2325 echo -n " ${qaStatus}" >> ${outfile}
2326
2327
2328 #fill OK/BAD rates
2329 for i in $(seq 0 $((${Nrates}-1))) ; do
2330 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2331 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2332
2333 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2334 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2335 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2336 fi
2337 echo -n " ${var1}" >> ${outfile}
2338 echo -n " ${var2}" >> ${outfile}
2339 done
2340
2341 if [[ -f ${fileOCDB} ]] ; then
2342 #fill counter
2343 for i in $(seq 0 $((${Ncounter}-1))) ; do
2344 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2345 echo -n " ${var1:-"-1"}" >> ${outfile}
2346 done
2347
2348 #fill status
2349 for i in $(seq 0 $((${Nstatus}-1))) ; do
2350 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2351 echo -n " ${var1:-"-1"}" >> ${outfile}
2352 done
2353 fi
2354 echo >> ${outfile}
2355 done
2356
5de3064b 2357 return 0
78d0bbfd 2358)
71353a5f 2359
2360parseConfig()
2361{
2362 configFile=${1}
2363 shift
444b9916 2364 args=("$@")
2365
0eaf38c7 2366
444b9916 2367 #some defaults
6fc48f9e 2368 #autoOCDB=0
2369 defaultOCDB="raw://"
2370 #runNumber=167123
2371 #makeflowPath="/hera/alice/aux/cctools/bin"
2372 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2373 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2374 makeflowOptions=""
39527a15 2375 #batchCommand="/usr/bin/qsub"
6fc48f9e 2376 batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2377 baseOutputDirectory="$PWD/output"
2378 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2379 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2380 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2381 reconstructInTemporaryDir=0
2382 recoTriggerOptions="\"\""
2383 percentProcessedFilesToContinue=100
2384 maxSecondsToWait=$(( 3600*24 ))
2385 nEvents=-1
2386 nMaxChunks=0
2387 postSetUpActionCPass0=""
2388 postSetUpActionCPass1=""
2389 runCPass0reco=1
2390 runCPass0MergeMakeOCDB=1
2391 runCPass1reco=1
2392 runCPass1MergeMakeOCDB=1
2393 runESDfiltering=1
444b9916 2394 filteringFactorHighPt=1e2
2395 filteringFactorV0s=1e1
6fc48f9e 2396 MAILTO=""
2397 #pretend=1
2398 #dontRedirectStdOutToLog=1
2399 logToFinalDestination=1
2400 ALIROOT_FORCE_COREDUMP=1
0eaf38c7 2401 pretendDelay=0
71353a5f 2402
2403 #first, source the config file
2404 if [ -f ${configFile} ]; then
2405 source ${configFile}
2406 else
0eaf38c7 2407 echo "config file ${configFile} not found!"
2408 return 1
71353a5f 2409 fi
2410
839b07ca 2411 unset encodedSpaces
2412 for opt in "${args[@]}"; do
0eaf38c7 2413 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
839b07ca 2414 done
2415
d56eeaab 2416 #then, parse the options as they override the options from file
2417 for opt in "${args[@]}"; do
0eaf38c7 2418 [[ -z ${opt} ]] && continue
2419 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2420 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
d56eeaab 2421 if [[ ! "${opt}" =~ .*=.* ]]; then
0eaf38c7 2422 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
d56eeaab 2423 return 1
2424 fi
2425 local var="${opt%%=*}"
2426 local value="${opt#*=}"
9d0c204f 2427 echo "${var}=${value}"
444b9916 2428 export ${var}="${value}"
71353a5f 2429 done
2430
6fc48f9e 2431 #do some checking
2432 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2433
71353a5f 2434 #export the aliroot function if defined to override normal behaviour
2435 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
6fc48f9e 2436
5de3064b 2437 return 0
71353a5f 2438}
2439
2440aliroot()
2441{
e48b4924 2442 args=("$@")
6fc48f9e 2443 if [[ -n ${useProfilingCommand} ]]; then
e48b4924 2444 profilerLogFile="cpu.txt"
2445 [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2446 [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2447 echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}"
2448 ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
71353a5f 2449 else
2450 #to prevent an infinite recursion use "command aliroot" to disable
2451 #aliases and functions
e48b4924 2452 echo running command aliroot "${args[@]}"
2453 command aliroot "${args[@]}"
71353a5f 2454 fi
5de3064b 2455 return 0
71353a5f 2456}
2457
2458guessRunData()
2459{
2460 #guess the period from the path, pick the rightmost one
2461 period=""
2462 runNumber=""
2463 year=""
2464 pass=""
2465 legoTrainRunNumber=""
2466 dataType=""
2467
2468 local shortRunNumber=""
2469 local IFS="/"
2470 declare -a path=( $1 )
2471 local dirDepth=$(( ${#path[*]}-1 ))
2472 i=0
2473 #for ((x=${dirDepth};x>=0;x--)); do
2474 for ((x=0;x<=${dirDepth};x++)); do
2475
2476 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2477 local field=${path[${x}]}
2478 local fieldNext=${path[$((x+1))]}
2479
2480 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2481 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2482 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2483 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2484 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2485 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2486 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2487 (( i++ ))
2488 done
2489 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2490 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2491
2492 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2493 if [[ -z ${runNumber}} ]];
2494 then
2495 #error condition
2496 return 1
2497 else
2498 #ALL OK
2499 return 0
2500 fi
5de3064b 2501 return 0
71353a5f 2502}
2503
839b07ca 2504#these functions encode strings to and from a space-less form
2505#use when spaces are not well handled (e.g. in arguments to
2506#commands in makeflow files, etc.
0eaf38c7 2507encSpaces()(echo "${1// /@@@@}")
2508decSpaces()(echo "${1//@@@@/ }")
839b07ca 2509
fc7e1e45 2510main "$@"