small fixes
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
fc7e1e45 4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
71353a5f 10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
fc7e1e45 24 umask 0002
25 shift
71353a5f 26 case ${runMode} in
fc7e1e45 27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
71353a5f 29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 33 "MakeSummary") goMakeSummary "$@";;
cb473870 34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
fc7e1e45 36 "test") goTest "$@";;
71353a5f 37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
cb473870 39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 41 "Merge") goMerge "$@";;
42 *)
444b9916 43 ${runMode} "$@"
fc7e1e45 44 ;;
45 esac
39527a15 46 return 0
fc7e1e45 47}
48
71353a5f 49generateMC()
50{
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
54 runNumber=${1}
55 OCDBpath=${2}
56 nEventsim=${3}
57 if [[ -n ${pretend} ]]; then
58 touch galice.root
59 else
60 if [[ -f sim.C && -f Config.C ]] ; then
61 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
62 mv syswatch.log simwatch.log
63 fi
64 fi
65}
66
fc7e1e45 67goCPass0()
68{
69 umask 0002
70
71353a5f 71 targetDirectory=${1}
72 inputList=${2}
73 nEvents=${3}
74 ocdbPath=${4}
75 configFile=${5}
76 runNumber=${6}
77 jobindex=${7}
78 shift 7
6fc48f9e 79 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 80
39527a15 81 #record the working directory provided by the batch system
82 batchWorkingDirectory=${PWD}
83
71353a5f 84 #use the jobindex only if set and non-negative
85 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
86 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
87 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
88 fi
89
90 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
91 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
92
93 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
94
95 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
96 ulimit -c unlimited
97 export ALIROOT_FORCE_COREDUMP
98 fi
fc7e1e45 99
cb473870 100 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
101 #at the QA stage the pass number is guessed from the path stored here.
102 #The Format is:
103 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
104 export PRODUCTION_METADATA="OutputDir=cpass0"
105
5de3064b 106 if [[ "${inputList}" =~ \.root$ ]]; then
107 infile=${inputList}
108 else
109 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
110 fi
71353a5f 111 chunkName=${infile##*/}
71353a5f 112
113 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
114 mkdir -p ${outputDir}
115 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
116
117 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
118 runpath=${outputDir}
119 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
120 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
71353a5f 121 mkdir -p ${runpath}
122 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
123 cd ${runpath}
1a0e38e3 124 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
fc7e1e45 125
5de3064b 126 #runCPassX/C expects the raw chunk to be linked in the run dir
127 #despite it being accessed by the full path
128 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 129
71353a5f 130 #####MC
131 if [[ -n ${generateMC} ]]; then
132 olddir=${PWD}
133 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
134 simrunpath=${outputDirMC}
135 [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
136 [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
137 mkdir -p ${outputDirMC}
138 mkdir -p ${simrunpath}
139 cd ${simrunpath}
140
141 filesMC=(
39527a15 142 "${batchWorkingDirectory}/sim.C"
143 "${batchWorkingDirectory}/rec.C"
144 "${batchWorkingDirectory}/Config.C"
145 "${batchWorkingDirectory}/OCDB_*.root"
71353a5f 146 )
147 for file in ${filesMC[*]}; do
148 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
149 done
fc7e1e45 150
71353a5f 151 generateMC ${runNumber} ${ocdbPath} ${nEvents}
fc7e1e45 152
71353a5f 153 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
154 cd ${olddir}
155
156 ln -s ${outputDirMC}/* ${runpath}/
157
158 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
159 infile=""
160 fi
161 ######
162
163 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
164
165 logOutputDir=${runpath}
166 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
167 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
168 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
169 echo "${0} $*"
fc7e1e45 170
171 echo "#####################"
172 echo CPass0:
173 echo JOB setup
71353a5f 174 echo nEvents ${nEvents}
175 echo runNumber ${runNumber}
176 echo ocdbPath ${ocdbPath}
177 echo infile ${infile}
178 echo chunkName ${chunkName}
179 echo jobindex ${jobindex}
180 echo recoTriggerOptions ${recoTriggerOptions}
181 echo targetDirectory ${targetDirectory}
182 echo commonOutputPath ${commonOutputPath}
183 echo doneFile ${doneFile}
184 echo runpath ${runpath}
185 echo outputDir ${outputDir}
186 echo ALICE_ROOT ${ALICE_ROOT}
187 echo PWD ${PWD}
fc7e1e45 188 echo "########## ###########"
189
71353a5f 190 alirootInfo > ALICE_ROOT.log
fc7e1e45 191
192 filesCPass0=(
39527a15 193 "${batchWorkingDirectory}/runCPass0.sh"
194 "${batchWorkingDirectory}/recCPass0.C"
195 "${batchWorkingDirectory}/runCalibTrain.C"
196 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
197 "${batchWorkingDirectory}/OCDB.root"
71353a5f 198 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
199 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
200 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 201 )
202
203 for file in ${filesCPass0[*]}; do
204 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 205 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 206 done
207
71353a5f 208 echo "this directory (${PWD}) contents:"
209 /bin/ls
fc7e1e45 210 echo
211 chmod u+x runCPass0.sh
212
71353a5f 213 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
214
215 if [[ -n ${postSetUpActionCPass0} ]]; then
216 echo "running ${postSetUpActionCPass0}"
217 eval ${postSetUpActionCPass0}
fc7e1e45 218 fi
219
220 #run CPass0
71353a5f 221 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
222 if [[ -n ${pretend} ]]; then
cb473870 223 touch AliESDs.root
224 touch AliESDfriends.root
fc7e1e45 225 touch AliESDfriends_v1.root
226 touch rec.log
227 touch calib.log
228 else
71353a5f 229 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
230 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 231 fi
232
fc7e1e45 233 #move stuff to final destination
71353a5f 234 echo "this directory (${PWD}) contents:"
235 /bin/ls
fc7e1e45 236 echo
237
71353a5f 238 echo rm -f ./${chunkName}
239 rm -f ./${chunkName}
240 echo "cp --recursive ${runpath}/* ${outputDir}"
241 cp -p --recursive ${runpath}/* ${outputDir}
fc7e1e45 242 echo
5ed7c82c 243
244 #validate CPass0
245 cd ${outputDir}
71353a5f 246 touch ${doneFile}
71353a5f 247 echo "dir ${outputDir}" >> ${doneFile}
ab946185 248 if summarizeLogs >> ${doneFile}; then
249 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
250 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
251 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
252 fi
fc7e1e45 253
71353a5f 254 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 255 return 0
fc7e1e45 256}
257
258goCPass1()
259{
260 umask 0002
261
71353a5f 262 targetDirectory=${1}
263 inputList=${2}
264 nEvents=${3}
265 ocdbPath=${4}
266 configFile=${5}
267 runNumber=${6}
268 jobindex=${7}
269 shift 7
444b9916 270 extraOpts=("$@")
6fc48f9e 271 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 272
39527a15 273 #record the working directory provided by the batch system
274 batchWorkingDirectory=${PWD}
275
71353a5f 276 #use the jobindex only if set and non-negative
277 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
278 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
279 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
280 fi
281
282 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
283 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
284
285 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
286
287 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
288 ulimit -c unlimited
289 export ALIROOT_FORCE_COREDUMP
290 fi
fc7e1e45 291
cb473870 292 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
293 #at the QA stage the pass number is guessed from the path stored here.
294 #The Format is:
295 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
296 export PRODUCTION_METADATA="OutputDir=cpass1"
297
71353a5f 298 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
fc7e1e45 299 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 300 infile=${inputList}
fc7e1e45 301 else
71353a5f 302 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 303 fi
fc7e1e45 304 chunkName=${infile##*/}
5de3064b 305
71353a5f 306 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
307 mkdir -p ${outputDir}
308 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
fc7e1e45 309
71353a5f 310 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
311 runpath=${outputDir}
312 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
313 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
314
315 #MC
316 if [[ "${infile}" =~ galice\.root ]]; then
317 ln -s ${inputList%/*}/* ${runpath}
318 infile=""
319 fi
fc7e1e45 320
321 #init the running path
71353a5f 322 mkdir -p ${runpath}
323 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
324 cd ${runpath}
1a0e38e3 325 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
fc7e1e45 326
71353a5f 327 #this is needed for runCPass1.sh
328 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 329
71353a5f 330 logOutputDir=${runpath}
331 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
332 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
333 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
334 echo "${0} $*"
fc7e1e45 335
336 echo "#####################"
337 echo CPass1:
338 echo JOB setup
71353a5f 339 echo nEvents ${nEvents}
340 echo runNumber ${runNumber}
341 echo ocdbPath ${ocdbPath}
342 echo infile ${infile}
343 echo chunkName ${chunkName}
344 echo jobindex ${jobindex}
345 echo recoTriggerOptions ${recoTriggerOptions}
346 echo targetDirectory ${targetDirectory}
347 echo commonOutputPath ${commonOutputPath}
348 echo doneFile ${doneFile}
349 echo runpath ${runpath}
350 echo outputDir ${outputDir}
351 echo ALICE_ROOT ${ALICE_ROOT}
352 echo PWD ${PWD}
fc7e1e45 353 echo "########## ###########"
354
71353a5f 355 alirootInfo > ALICE_ROOT.log
fc7e1e45 356
357 filesCPass1=(
39527a15 358 "${batchWorkingDirectory}/runCPass1.sh"
359 "${batchWorkingDirectory}/recCPass1.C"
360 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
361 "${batchWorkingDirectory}/runCalibTrain.C"
362 "${batchWorkingDirectory}/QAtrain_duo.C"
363 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
617f8886 364 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 365 "${batchWorkingDirectory}/OCDB.root"
71353a5f 366 "${trustedQAtrainMacro}"
367 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
368 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
369 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
370 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
371 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 372 )
373
374 for file in ${filesCPass1[*]}; do
375 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 376 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 377 done
378
71353a5f 379 echo "this directory (${PWD}) contents:"
380 /bin/ls
fc7e1e45 381 echo
382
71353a5f 383 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
384
385 if [[ -n ${postSetUpActionCPass1} ]]; then
386 echo "running ${postSetUpActionCPass1}"
387 eval ${postSetUpActionCPass1}
fc7e1e45 388 echo
389 fi
390
391 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
392 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 393 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
394 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 395 else
396 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
397 fi
398
71353a5f 399 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
cb473870 400 echo "cpass0 produced no calibration! exiting..."
71353a5f 401 touch ${doneFile}
402 return 1
cb473870 403 fi
fc7e1e45 404
71353a5f 405 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
406 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
407 #entries using a relative path, e.g. local://./OCDB
408 mkdir Barrel OuterDet
409 ln -s ../OCDB Barrel/OCDB
410 ln -s ../OCDB OuterDet/OCDB
411
444b9916 412 #setup the filtering
413 #the following option enables the filtering task inside the QAtrain_duo.C
414 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
415 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
416 if [[ -n ${filteringFactorHighPt} ]]; then
417 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
418 fi
419 if [[ -n ${filteringFactorV0s} ]]; then
420 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
421 fi
422
fc7e1e45 423 #run CPass1
424 chmod u+x runCPass1.sh
71353a5f 425 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
426 if [[ -n ${pretend} ]]; then
cb473870 427 touch AliESDs_Barrel.root
428 touch AliESDfriends_Barrel.root
fc7e1e45 429 touch AliESDfriends_v1.root
cb473870 430 touch QAresults_barrel.root
431 touch EventStat_temp_barrel.root
432 touch AODtpITS.root
433 touch AliESDs_Outer.root
434 touch AliESDfriends_Outer.root
435 touch QAresults_outer.root
436 touch EventStat_temp_outer.root
fc7e1e45 437 touch rec.log
438 touch calib.log
439 touch qa.log
444b9916 440 touch filtering.log FilterEvents_Trees.root
fc7e1e45 441 else
71353a5f 442 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 443
444 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
445 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
446 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
447 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
448 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
449 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
450 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
451 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
452
453 #make the filtered tree (if requested and not already produced by QA
454 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
455 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
456 goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
457 fi
458
fc7e1e45 459 fi
cb473870 460
461 ##handle possible crashes in QA (happens often in trunk)
462 ##rerun QA with a trusted aliroot version
463 #if [[ $(validateLog qa_barrel.log) ]]; then
464 # echo "qa_barrel.log not validated!"
465 #fi
466 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 467 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 468 # source ${setupTrustedAliROOTenvInCurrentShell}
469 # cd Barrel
470 # rm QAresults_barrel.root
471 # rm EventStat_temp_barrel.root
472 # rm AODtpITS.root
473 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
474 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 475 # if [[ -n ${trustedQAtrainMacro} ]]; then
476 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 477 # fi
71353a5f 478 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
479 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 480 # cd ../
481 #fi
cb473870 482
fc7e1e45 483 #move stuff to final destination
71353a5f 484 echo "this directory (${PWD}) contents:"
485 /bin/ls
486 echo rm -f ./${chunkName}
487 rm -f ./${chunkName}
fc7e1e45 488 echo "cp --recursive ${runpath}/* ${outputDir}"
71353a5f 489 cp -pf --recursive ${runpath}/* ${outputDir}
fc7e1e45 490 echo
491
5ed7c82c 492 #validate CPass1
493 cd ${outputDir}
71353a5f 494 touch ${doneFile}
71353a5f 495 echo "dir ${outputDir}" >> ${doneFile}
ab946185 496 if summarizeLogs >> ${doneFile}; then
497 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
498 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
499 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
500 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
501 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
502 fi
503
71353a5f 504 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 505 return 0
fc7e1e45 506}
507
508
509goMergeCPass0()
510{
fc7e1e45 511 #
512 # find the output files and merge them
513 #
514
71353a5f 515 outputDir=${1}
516 ocdbStorage=${2}
517 configFile=${3}
518 runNumber=${4}
519 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
520 shift 5
6fc48f9e 521 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 522
39527a15 523 #record the working directory provided by the batch system
524 batchWorkingDirectory=${PWD}
525
71353a5f 526 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
527 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
528
529 umask 0002
530 ulimit -c unlimited
fc7e1e45 531
71353a5f 532 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 533
71353a5f 534 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
535 runpath=${outputDir}
536 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
537 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
fc7e1e45 538
71353a5f 539 mkdir -p ${runpath}
540 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
541 cd ${runpath}
1a0e38e3 542 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
fc7e1e45 543
71353a5f 544 logOutputDir=${runpath}
545 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
546 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
547 echo "${0} $*"
fc7e1e45 548
fc7e1e45 549 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 550
551 echo goMergeCPass0 SETUP:
71353a5f 552 echo runNumber=${runNumber}
553 echo outputDir=${outputDir}
554 echo ocdbStorage=${ocdbStorage}
555 echo calibrationFilesToMerge=${calibrationFilesToMerge}
556 echo mergingScript=${mergingScript}
557 echo commonOutputPath=${commonOutputPath}
558 echo runpath=${runpath}
fc7e1e45 559
560 # copy files in case they are not already there
561 filesMergeCPass0=(
39527a15 562 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
563 "${batchWorkingDirectory}/OCDB.root"
564 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 565 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
566 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
567 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
568 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
569 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 570 )
571 for file in ${filesMergeCPass0[*]}; do
572 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 573 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 574 done
575
71353a5f 576 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
577
578 alirootInfo > ALICE_ROOT.log
fc7e1e45 579
580 #
cb473870 581 echo "PWD"
71353a5f 582 /bin/ls
cb473870 583 echo "PWD/.."
71353a5f 584 /bin/ls ../
cb473870 585
fc7e1e45 586
587 #merge calibration
71353a5f 588 chmod u+x ${mergingScript}
fc7e1e45 589 mkdir -p ./OCDB
71353a5f 590 if [[ ! -f ${calibrationFilesToMerge} ]]; then
591 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
592 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 593 fi
594
71353a5f 595 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
596 if [[ -n ${pretend} ]]; then
fc7e1e45 597 touch CalibObjects.root
598 touch ocdb.log
599 touch merge.log
617f8886 600 touch dcsTime.root
71353a5f 601 mkdir -p ./OCDB/TPC/Calib/TimeGain/
602 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
603 touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
604 touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 605 else
71353a5f 606 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
fc7e1e45 607
617f8886 608 #produce the calib trees for expert QA (dcsTime.root)
609 goMakeLocalOCDBaccessConfig ./OCDB
610 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
611 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
612 fi
613
fc7e1e45 614 ### produce the output
615 #tar the produced OCDB for reuse
71353a5f 616 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
fc7e1e45 617
71353a5f 618 /bin/ls
5ed7c82c 619
620 #copy all to output dir
71353a5f 621 cp -pf --recursive ${runpath}/* ${outputDir}
617f8886 622
71353a5f 623 if [[ -n ${generateMC} ]]; then
624 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
625 fi
626
fc7e1e45 627 #validate merging cpass0
5ed7c82c 628 cd ${outputDir}
71353a5f 629 touch ${doneFile}
71353a5f 630 echo "dir ${outputDir}" >> ${doneFile}
ab946185 631 if summarizeLogs >> ${doneFile}; then
632 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
633 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
634 fi
fc7e1e45 635
71353a5f 636 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 637 return 0
fc7e1e45 638}
639
640goMergeCPass1()
641{
fc7e1e45 642 #
643 # find the output files and merge them
644 #
645
71353a5f 646 outputDir=${1}
647 ocdbStorage=${2}
648 configFile=${3}
649 runNumber=${4}
650 calibrationFilesToMerge=${5}
651 qaFilesToMerge=${6}
444b9916 652 filteredFilesToMerge=${7}
653 shift 7
6fc48f9e 654 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 655
39527a15 656 #record the working directory provided by the batch system
657 batchWorkingDirectory=${PWD}
658
71353a5f 659 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
660 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
661
662 umask 0002
663 ulimit -c unlimited
fc7e1e45 664
665 #clean up first:
71353a5f 666 rm -f ${outputDir}/*.log
667 rm -f ${outputDir}/*.root
668 rm -f ${outputDir}/*done
669
670 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
671
672 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
673 runpath=${outputDir}
674 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
675 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
676
677 mkdir -p ${runpath}
678 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
679 cd ${runpath}
1a0e38e3 680 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
71353a5f 681
682 logOutputDir=${runpath}
683 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
684 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
685 echo "${0} $*"
686
687 calibrationOutputFileName='AliESDfriends_v1.root'
688 qaOutputFileName='QAresults*.root'
fc7e1e45 689 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 690 #important to have the string "Stage.txt" in the filename to trigger the merging
691 #it has to be a list of directories containing the files
fc7e1e45 692 qaMergedOutputFileName="QAresults_merged.root"
693
694 echo goMergeCPass1 SETUP:
71353a5f 695 echo runNumber=${runNumber}
696 echo outputDir=${outputDir}
697 echo ocdbStorage=${ocdbStorage}
698 echo calibrationFilesToMerge=$calibrationFilesToMerge
699 echo qaFilesToMerge=$qaFilesToMerge
700 echo calibrationOutputFileName=${calibrationOutputFileName}
701 echo mergingScript=${mergingScript}
fc7e1e45 702
703 # copy files in case they are not already there
704 filesMergeCPass1=(
39527a15 705 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
706 "${batchWorkingDirectory}/${qaFilesToMerge}"
707 "${batchWorkingDirectory}/OCDB.root"
708 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
709 "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
710 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 711 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
712 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
713 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
714 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
715 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
716 "${trustedQAtrainMacro}"
717 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 718 )
719 for file in ${filesMergeCPass1[*]}; do
720 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 721 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 722 done
723
71353a5f 724 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
725
cb473870 726 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
727 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 728 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
729 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 730 else
731 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
732 fi
733
71353a5f 734 alirootInfo > ALICE_ROOT.log
fc7e1e45 735
736 #
71353a5f 737 /bin/ls
fc7e1e45 738
739 #merge calibration
71353a5f 740 chmod u+x ${mergingScript}
fc7e1e45 741 mkdir -p OCDB
444b9916 742
743 #if not provided, create the lists of files to merge
744 if [[ ! -f ${filteredFilesToMerge} ]]; then
745 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
746 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
747 fi
71353a5f 748 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 749 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
750 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
751 fi
752 if [[ ! -f ${qaFilesToMerge} ]]; then
753 #find the files, but only store the directories (QAtrain_duo.C requires this)
754 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
755 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 756 fi
757
71353a5f 758 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
759 if [[ -n ${pretend} ]]; then
fc7e1e45 760 touch ocdb.log
71353a5f 761 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 762 touch ${qaMergedOutputFileName}
fc7e1e45 763 touch merge.log
cb473870 764 touch trending.root
444b9916 765 touch FilterEvents_Trees.root
ab946185 766 touch CalibObjects.root
767 touch dcsTime.root
768 touch ${qaMergedOutputFileName}
617f8886 769 mkdir -p OCDB
fc7e1e45 770 else
617f8886 771 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
772
773 #merge QA (and filtered trees)
774 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
775 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
776
777 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
778 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 779 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
780 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
781 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 782 mv trending_barrel.root trending.root
444b9916 783
784 #merge filtered trees
785 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
786 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
617f8886 787
788 #produce the calib trees for expert QA
789 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
790 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
fc7e1e45 791 fi
617f8886 792
793 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
71353a5f 794
795 /bin/ls
5ed7c82c 796
797 #copy all to output dir
71353a5f 798 cp -pf --recursive ${runpath}/* ${outputDir}
5ed7c82c 799
fc7e1e45 800 #validate merge cpass1
5ed7c82c 801 cd ${outputDir}
71353a5f 802 touch ${doneFile}
71353a5f 803 echo "dir ${outputDir}" >> ${doneFile}
ab946185 804 if summarizeLogs >> ${doneFile}; then
805 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
806 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
807 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
808 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
809 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
810 fi
71353a5f 811
812 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 813 return 0
cb473870 814}
815
816goMerge()
817{
818 #generic root merge using CPass1 merge.C script
71353a5f 819 inputList=${1}
820 outputFile=${2}
cb473870 821 configFile=${3-"becnhmark.config"}
71353a5f 822 shift 3
6fc48f9e 823 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 824
825 #record the working directory provided by the batch system
826 batchWorkingDirectory=${PWD}
fc7e1e45 827
71353a5f 828 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
829 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
830 umask 0002
831 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
832 rm -f ${outputFile}
833 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 834 return 0
5dfac970 835}
fc7e1e45 836
cb473870 837goSubmitMakeflow()
5dfac970 838{
cb473870 839 #run
71353a5f 840 productionID=${1}
841 inputFileList=${2}
842 configFile=${3}
843 shift 3
444b9916 844 extraOpts=("$@")
6fc48f9e 845 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 846
847 #record the working directory provided by the batch system
848 batchWorkingDirectory=${PWD}
fc7e1e45 849
5dfac970 850 [[ -z ${configFile} ]] && configFile="benchmark.config"
851 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 852
71353a5f 853 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 854 echo "setting the makflow path from the config: "
855 echo " export PATH=${makeflowPath}:${PATH}"
856 export PATH=${makeflowPath}:${PATH}
857 fi
858
39527a15 859 #create the common output dir and the meta dir
860 commonOutputPath=${baseOutputDirectory}/${productionID}
861 if [[ -d ${commonOutputPath} ]]; then
862 echo "output dir ${commonOutputPath} exists!"
863 return 1
864 else
865 mkdir -p ${commonOutputPath}
866 fi
867 mkdir -p ${commonOutputPath}/meta
64a8d0f6 868
869 self=$(readlink -f "${0}")
870
871 cp ${self} ${commonOutputPath}
872 cp ${configFile} ${commonOutputPath}
873 cp ${inputList} ${commonOutputPath}
39527a15 874
cb473870 875 #submit - use makeflow if available, fall back to old stuff when makeflow not there
876 if which makeflow; then
cb473870 877
39527a15 878 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
71353a5f 879
cb473870 880 makeflow ${makeflowOptions} benchmark.makeflow
881 cd ../
882 else
883 echo "no makeflow!"
884 fi
5de3064b 885 return 0
5dfac970 886}
fc7e1e45 887
5dfac970 888goGenerateMakeflow()
889{
890 #generate the makeflow file
ab946185 891 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 892 productionID=${1}
893 inputFileList=${2}
894 configFile=${3}
895 shift 3
444b9916 896 extraOpts=("$@")
39527a15 897 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
898
1a0e38e3 899 #extra safety
900 if [[ -z ${commonOutputPath} ]]; then
901 commonOutputPath=${baseOutputDirectory}/${productionID}
902 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
903 fi
bff71da0 904
39527a15 905 #record the working directory provided by the batch system
906 batchWorkingDirectory=${PWD}
fc7e1e45 907
5dfac970 908 [[ -z ${configFile} ]] && configFile="benchmark.config"
909 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 910
5dfac970 911 #these files will be made a dependency - will be copied to the working dir of the jobs
912 declare -a copyFiles
913 inputFiles=(
914 "OCDB.root"
915 "localOCDBaccessConfig.C"
cb473870 916 "QAtrain_duo.C"
917 "runCPass1.sh"
918 "recCPass1.C"
919 "recCPass1_OuterDet.C"
920 "runCalibTrain.C"
921 "runCPass0.sh"
922 "recCPass0.C"
5dfac970 923 )
924 for file in ${inputFiles[*]}; do
925 [[ -f ${file} ]] && copyFiles+=("${file}")
926 done
927
928 #create the makeflow file
cb473870 929 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 930 declare -A arr_cpass0_merged arr_cpass1_merged
931 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
932 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 933 declare -A arr_cpass0_profiled_outputs
444b9916 934 declare -A listOfRuns
935 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 936 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 937 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 938 [[ -z ${runNumber} ]] && continue
fc7e1e45 939 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 940
444b9916 941 unset arr_cpass0_outputs
942 unset arr_cpass1_outputs
943 declare -a arr_cpass0_outputs
944 declare -a arr_cpass1_outputs
fc7e1e45 945
cb473870 946 jobindex=0
6fc48f9e 947 inputFile=""
5dfac970 948 while read inputFile; do
949 currentDefaultOCDB=${defaultOCDB}
cb473870 950 [[ -z ${autoOCDB} ]] && autoOCDB=1
951 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 952 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 953 fi
fc7e1e45 954
5dfac970 955 #CPass0
39527a15 956 arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
444b9916 957 echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
958 echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
39527a15 959 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
fc7e1e45 960 echo
5dfac970 961
962 #CPass1
39527a15 963 arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
964 echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
444b9916 965 echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
39527a15 966 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
fc7e1e45 967 echo
5dfac970 968 ((jobindex++))
fc7e1e45 969
71353a5f 970 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 971
5dfac970 972 #CPass0 list of Calib files to merge
39527a15 973 arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
444b9916 974 echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
975 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
71353a5f 976 echo
977
978 #CPass0 merging
39527a15 979 arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
980 echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
444b9916 981 echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
39527a15 982 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
5dfac970 983 echo
fc7e1e45 984
ab946185 985 #CPass1 list of Calib/QA/ESD/filtered files
986 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
987 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 988 # this corresponds to the last merging stage)
39527a15 989 arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
444b9916 990 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
991 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
5dfac970 992 echo
39527a15 993 arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
ab946185 994 echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
995 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
996 echo
39527a15 997 arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
444b9916 998 echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
999 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1000 echo
39527a15 1001 arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
444b9916 1002 echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1003 echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
ab946185 1004 echo
444b9916 1005
71353a5f 1006 #CPass1 merging
39527a15 1007 arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1008 echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
444b9916 1009 echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
39527a15 1010 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
5dfac970 1011 echo
fc7e1e45 1012
6fc48f9e 1013 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1014 if [[ -n ${profilingCommand} ]]; then
1015 arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
6fc48f9e 1016 echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1017 echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
39527a15 1018 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
6fc48f9e 1019 echo "\"useProfilingCommand=${profilingCommand}\""
1020 echo
1021 fi
1022
444b9916 1023 done #runs
fc7e1e45 1024
5dfac970 1025 #Summary
1a0e38e3 1026 echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
39527a15 1027 echo -n " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1028 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
cb473870 1029 echo
1030
5de3064b 1031 return 0
cb473870 1032}
1033
1034goPrintValues()
1035{
1036 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1037 if [[ $# -lt 3 ]]; then
1038 echo "goPrintValues key outputFile inputFiles"
1039 echo "if outputFile is \"-\" dont redirect to a file"
1040 return
1041 fi
1042 key=${1}
1043 outputFile=${2}
1044 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1045 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1046 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1047 return 0
5dfac970 1048}
fc7e1e45 1049
5dfac970 1050goCreateQAplots()
1051{
1052 umask 0002
71353a5f 1053 mergedQAfileList=${1}
1054 productionID=${2}
1055 outputDir=${3}
1056 configFile=${4}
1057 shift 4
6fc48f9e 1058 if ! parseConfig ${configFile} ${@}; then return 1; fi
71353a5f 1059
39527a15 1060 #record the working directory provided by the batch system
1061 batchWorkingDirectory=${PWD}
1062
71353a5f 1063 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1064
71353a5f 1065 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1066 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1067 echo "${0} $*"
1068
1069 olddir=${PWD}
1070 mkdir -p ${outputDir}
1071 cd ${outputDir}
64a8d0f6 1072 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1073
9510565c 1074 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
fc7e1e45 1075
71353a5f 1076 cd ${olddir}
5de3064b 1077 return 0
5dfac970 1078}
fc7e1e45 1079
5dfac970 1080goTest()
1081{
1082 umask 0002
1083 exec 2>&1
1084 exec > >(tee test.log)
1085 echo "$@"
1086 echo something
5de3064b 1087 return 0
5dfac970 1088}
fc7e1e45 1089
5dfac970 1090alirootInfo()
1091{
5dfac970 1092 umask 0002
71353a5f 1093 # save aliroot repository info
1094 [[ -z "${ALICE_ROOT}" ]] && return 1
1095
1096 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1097 echo "\${ROOTSYS}=${ROOTSYS}"
1098 echo "\${PATH}=${PATH}"
1099 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1100
1101 pushd ${PWD}
1102 cd ${ALICE_ROOT}
5dfac970 1103 echo
71353a5f 1104
1105 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1106 git status
5dfac970 1107 echo ""
1108 echo ""
71353a5f 1109 git diff ${currentBranch}
1110 popd
5de3064b 1111 return 0
5dfac970 1112}
fc7e1e45 1113
5dfac970 1114setYear()
1115{
1116 #set the year
71353a5f 1117 # ${1} - year to be set
1118 # ${2} - where to set the year
1119 year1=$(guessYear ${1})
1120 year2=$(guessYear ${2})
1121 local path=${2}
1122 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1123 echo ${path}
5de3064b 1124 return 0
5dfac970 1125}
fc7e1e45 1126
5dfac970 1127guessPeriod()
1128{
1129 #guess the period from the path, pick the rightmost one
1130 local IFS="/"
71353a5f 1131 declare -a path=( ${1} )
5dfac970 1132 local dirDepth=${#path[*]}
1133 for ((x=${dirDepth}-1;x>=0;x--)); do
1134 local field=${path[${x}]}
71353a5f 1135 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1136 done
71353a5f 1137 echo ${period}
5de3064b 1138 return 0
5dfac970 1139}
fc7e1e45 1140
5dfac970 1141guessYear()
1142{
1143 #guess the year from the path, pick the rightmost one
1144 local IFS="/"
71353a5f 1145 declare -a path=( ${1} )
5dfac970 1146 local dirDepth=${#path[*]}
1147 for ((x=${dirDepth}-1;x>=0;x--)); do
1148 local field=${path[${x}]}
71353a5f 1149 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1150 done
71353a5f 1151 echo ${year}
5de3064b 1152 return 0
5dfac970 1153}
fc7e1e45 1154
5dfac970 1155guessRunNumber()
1156{
1157 #guess the run number from the path, pick the rightmost one
71353a5f 1158 #works for /path/foo/000123456/bar/...
1159 #and /path/foo.run123456.bar
1160 local IFS="/."
1161 declare -a path=( ${1} )
5dfac970 1162 local dirDepth=${#path[*]}
1163 for ((x=${dirDepth}-1;x>=0;x--)); do
1164 local field=${path[${x}]}
71353a5f 1165 field=${field/run/000}
617f8886 1166 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1167 done
71353a5f 1168 echo ${runNumber}
5de3064b 1169 return 0
5dfac970 1170}
fc7e1e45 1171
cb473870 1172validateLog()
1173{
71353a5f 1174 log=${1}
cb473870 1175 errorConditions=(
71353a5f 1176 'There was a crash'
1177 'floating'
1178 'error while loading shared libraries'
1179 'std::bad_alloc'
1180 's_err_syswatch_'
1181 'Thread [0-9]* (Thread'
1182 'AliFatal'
1183 'core dumped'
1184 '\.C.*error:.*\.h: No such file'
64a8d0f6 1185 'line.*Aborted'
71353a5f 1186 )
1187
1188 warningConditions=(
1189 'This is serious !'
1190 'rocVoltage out of range:'
cb473870 1191 )
1192
71353a5f 1193 local logstatus=0
1194 local errorSummary=""
1195 local warningSummary=""
1196
1197 for ((i=0; i<${#errorConditions[@]};i++)); do
1198 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1199 [[ -n ${tmp} ]] && tmp+=" : "
1200 errorSummary+=${tmp}
1201 done
1202
1203 for ((i=0; i<${#warningConditions[@]};i++)); do
1204 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1205 [[ -n ${tmp} ]] && tmp+=" : "
1206 warningSummary+=${tmp}
1207 done
1208
1209 if [[ -n ${errorSummary} ]]; then
1210 echo "${errorSummary}"
1211 return 1
1212 fi
cb473870 1213
71353a5f 1214 if [[ -n ${warningSummary} ]]; then
1215 echo "${warningSummary}"
1216 return 2
1217 fi
1218
1219 return 0
cb473870 1220}
1221
5dfac970 1222summarizeLogs()
1223{
1224 #print a summary of logs
1225 logFiles=(
1226 "*.log"
cb473870 1227 "stdout"
1228 "stderr"
5dfac970 1229 )
fc7e1e45 1230
cb473870 1231 #check logs
71353a5f 1232 local logstatus=0
5dfac970 1233 for log in ${logFiles[*]}; do
71353a5f 1234 finallog=${PWD%/}/${log}
1235 [[ ! -f ${log} ]] && continue
1236 errorSummary=$(validateLog ${log})
1237 validationStatus=$?
1238 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1239 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1240 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1241 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1242 echo "${finallog} OK"
fc7e1e45 1243 else
71353a5f 1244 echo "${finallog} OK"
fc7e1e45 1245 fi
71353a5f 1246 elif [[ ${validationStatus} -eq 1 ]]; then
1247 echo "${finallog} BAD ${errorSummary}"
ab946185 1248 logstatus=1
71353a5f 1249 elif [[ ${validationStatus} -eq 2 ]]; then
1250 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1251 fi
1252 done
cb473870 1253
1254 #report core files
71353a5f 1255 while read x; do
1256 echo ${x}
1257 chmod 644 ${x}
1258 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1259 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1260
71353a5f 1261 return ${logstatus}
5dfac970 1262}
fc7e1e45 1263
5dfac970 1264spitOutLocalOCDBaccessConfig()
1265{
1266 umask 0002
71353a5f 1267 #find ${1} -name "*root" | \
1268 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1269 while read line
1270 do
71353a5f 1271 local tmp=${line#${1}}
5dfac970 1272 echo ${tmp%/*} | \
71353a5f 1273 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1274 done
5de3064b 1275 return 0
5dfac970 1276}
fc7e1e45 1277
71353a5f 1278goMakeLocalOCDBaccessConfig()
5dfac970 1279{
1280 umask 0002
1281 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1282 local localOCDBpathCPass0=${1}
1283 local OCDBpathPrefix=${2}
1284 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1285
1286 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1287 tar xzf ${localOCDBpathCPass0}
1288 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1289 fi
fc7e1e45 1290
5dfac970 1291 echo
1292 echo creating the specific storage script
1293 echo localOCDBaccessConfig.C
71353a5f 1294 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1295 echo
fc7e1e45 1296
5dfac970 1297 local tempLocalOCDB=""
1298 if [[ -f localOCDBaccessConfig.C ]]; then
1299 tempLocalOCDB=$(mktemp)
71353a5f 1300 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1301 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1302 fi
fc7e1e45 1303
5dfac970 1304 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1305 echo "{" >> localOCDBaccessConfig.C
1306 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1307 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1308 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1309 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1310
71353a5f 1311 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1312
5dfac970 1313 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1314 echo
1315 echo "!!!!!!! CPass0 produced no OCDB entries"
1316 return 1
1317 fi
5de3064b 1318 return 0
5dfac970 1319}
fc7e1e45 1320
cb473870 1321goMakeFilteredTrees()
1322{
71353a5f 1323 outputDir=${1}
1324 runNumber=${2}
cb473870 1325 #get path to input list
71353a5f 1326 inputListfiles=${3}
cb473870 1327 #get scale number for tracks
71353a5f 1328 filterT=${4}
cb473870 1329 #get scale number for V0s
71353a5f 1330 filterV=${5}
cb473870 1331 #get OCDB path (optional)
1332 OCDBpath=${6}
1333 #get max number of files
1334 maxFiles=${7-"1000000"}
1335 #get offset of first file
1336 offsetFile=${8-"0"}
1337 #get max number of events
1338 maxEvents=${9-"30000000"}
1339 #get offset of first event
1340 offsetEvent=${10-"0"}
1341 configFile=${11-"benchmark.config"}
1342 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1343 shift 12
6fc48f9e 1344 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1345
1346 #record the working directory provided by the batch system
1347 batchWorkingDirectory=${PWD}
1348
cb473870 1349
1350 commonOutputPath=${PWD}
71353a5f 1351 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1352
444b9916 1353 runpath=${outputDir}
1354 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1355 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1356
cb473870 1357
71353a5f 1358 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1359
cb473870 1360 mkdir -p ${outputDir}
1361 mkdir -p ${runpath}
1362
1363 cd ${runpath}
1a0e38e3 1364 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
cb473870 1365
1366 cat > filtering.log << EOF
1367 goMakeFilteredTrees config:
71353a5f 1368 runpath=${runpath}
1369 outputDir=${outputDir}
1370 commonOutputPath=${commonOutputPath}
1371 ALICE_ROOT=${ALICE_ROOT}
1372 PATH=${PATH}
1373 offsetEvent=$offsetEvent
1374 configFile=$configFile
1375 esdFileName=$esdFileName
cb473870 1376EOF
1377
1378 if [[ -z ${pretend} ]];then
71353a5f 1379 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1380 else
71353a5f 1381 touch filtering.log FilterEvents_Trees.root
cb473870 1382 fi
1383 pwd
71353a5f 1384 /bin/ls
1385 touch ${doneFile}
1386 summarizeLogs >> ${doneFile}
cb473870 1387 echo mv -f * ${outputDir}
1388 mv -f * ${outputDir}
71353a5f 1389 [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
cb473870 1390 cd ${commonOutputPath}
71353a5f 1391 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
5de3064b 1392 return 0
cb473870 1393}
1394
1395submit()
1396{
1397 umask 0002
444b9916 1398 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1399 JobID=${1}
1400 startID=${2}
1401 endID=${3}
1402 waitForJOBID=${4}
1403 command=${5}
444b9916 1404 shift 5
1405 commandArgs=("$@")
1406
1407 #add quote strings around the extra arguments
1408 for ((i=0; i<${#commandArgs[@]}; i++)); do
1409 commandArgs[i]=\"${commandArgs[i]}\"
1410 done
cb473870 1411
71353a5f 1412 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1413
1414 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1415
1416 batchSystem="SGE"
1417
71353a5f 1418 if [[ -z "${newFarm}" ]]
cb473870 1419 then
1420 #old LSF
1421 # submit it (as job array)
71353a5f 1422 nFiles=$(( ${endID}-${startID}+1 ))
1423 while [ ${startID} -le ${nFiles} ] ; do
1424 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1425 endID=$(expr ${startID} + 999)
cb473870 1426 else
71353a5f 1427 endID=${nFiles}
cb473870 1428 fi
1429 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1430 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1431 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1432 else
71353a5f 1433 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1434 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1435 fi
71353a5f 1436 startID=$(expr ${endID} + 1)
cb473870 1437 done
1438 else
1439 #new SGE farm
1440 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1441 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1442 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1443 else
444b9916 1444 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1445 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1446 fi
1447 fi
5de3064b 1448 return 0
cb473870 1449}
1450
1451goSubmitBatch()
5dfac970 1452{
71353a5f 1453 if [[ $# -lt 3 ]]; then
1454 echo "minimal use:"
1455 echo " ${0} submit fileList productionID configFile"
1456 return 0
1457 fi
1458
1459 productionID=${1}
1460 inputList=${2}
1461 configFile=${3:-"benchmark.config"}
1462 configFile=$(readlink -f ${configFile})
1463 shift 3
444b9916 1464 extraOpts=("$@")
6fc48f9e 1465 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1466
1467 #record the working directory provided by the batch system
1468 batchWorkingDirectory=${PWD}
fc7e1e45 1469
5dfac970 1470 #redirect all output to submit.log
1471 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1472 exec 7>&1
1473 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1474
5dfac970 1475 umask 0002
71353a5f 1476 echo ${0}" submit $*"
1477 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1478 then
fc7e1e45 1479 echo
71353a5f 1480 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1481 echo
71353a5f 1482 return
fc7e1e45 1483 fi
fc7e1e45 1484
5dfac970 1485 # check if config file is there
71353a5f 1486 if [ ! -f ${configFile} ]; then
1487 echo "ERROR! Config File '${configFile}' not found" >&2
1488 return
5dfac970 1489 else
71353a5f 1490 echo "Using Config File: '${configFile}'"
fc7e1e45 1491 fi
1492
9510565c 1493 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1494
71353a5f 1495 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1496 self=$(readlink -f "${0}")
1497 configPath=$(dirname ${configFile})
5dfac970 1498 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1499
cb473870 1500 mkdir -p ${commonOutputPath}
71353a5f 1501 mkdir -p ${commonOutputPath}/logs
1502 mkdir -p ${commonOutputPath}/meta
1503
1504 cp ${self} ${commonOutputPath}
1505 cp ${configFile} ${commonOutputPath}
1506 cp ${inputList} ${commonOutputPath}
cb473870 1507 self=${commonOutputPath}/${self##*/}
71353a5f 1508 chmod u+x ${self}
cb473870 1509 configFile=${commonOutputPath}/${configFile##*/}
1510 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1511
5dfac970 1512 #convert to absolut pathnames
71353a5f 1513 inputList=$(readlink -f "${inputList}")
5dfac970 1514 #make list of runs
71353a5f 1515 if [[ -z ${runNumber} ]]; then
1516 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1517 else
71353a5f 1518 listOfRuns=${runNumber}
5dfac970 1519 fi
fc7e1e45 1520
71353a5f 1521 alirootSource=$(readlink -f "${alirootSource}")
fc7e1e45 1522
5dfac970 1523 echo ""
1524 echo "### BEGIN CONFIGURATION ###"
1525 echo ""
1526 echo "GENERAL:"
1527 echo ""
71353a5f 1528 echo " productionID: ${productionID}"
1529 echo " batchCommand: ${batchCommand}"
1530 echo " batchFlags: ${batchFlags}"
9510565c 1531 echo " alirootEnv: ${alirootEnv}"
1532 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1533 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1534 echo " inputList: ${inputList}"
1535 echo " configPath: ${configPath}"
1536 echo " commonOutputPath: ${commonOutputPath}"
1537 echo " defaultOCDB: ${defaultOCDB}"
1538 echo " autoOCDB: ${autoOCDB}"
1539 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1540 echo " runs:"
1541 echo " ${listOfRuns[*]}"
1542 echo ""
1543 echo "THE TRAIN WILL RUN:"
fc7e1e45 1544
71353a5f 1545 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1546 echo " Pass0 - Recontruction"
1547 fi
fc7e1e45 1548
71353a5f 1549 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1550 echo " Pass0 - merging and OCDB export"
1551 fi
fc7e1e45 1552
71353a5f 1553 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1554 echo " Pass1 - Recontruction"
1555 fi
71353a5f 1556 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1557 echo " Pass1 - merging and OCDB export"
1558 fi
fc7e1e45 1559
5dfac970 1560 echo ""
1561 echo "LIMITS:"
71353a5f 1562 echo " max. Events/Chunk: ${nEvents}"
1563 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1564 echo ""
1565 echo "### END CONFIGURATION ###"
1566 echo ""
fc7e1e45 1567
36b5de1a 1568
5dfac970 1569 # check if input file is there
71353a5f 1570 if [ ! -f ${inputList} ]; then
1571 echo "ERROR! Input List '${inputList}' not found" >&2
1572 return
5dfac970 1573 fi
1574
1575 # define jobid (for dependent jobs)
cb473870 1576 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1577 #for each run we submit one jobarray:
fc7e1e45 1578 for runNumber in ${listOfRuns[*]}; do
cb473870 1579
71353a5f 1580 [[ -z ${runNumber} ]] && continue
fc7e1e45 1581 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1582
1583 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1584 JOBID1="p0_${JOBpostfix}"
1585 JOBID1wait="w0_${JOBpostfix}"
1586 JOBID2="m0_${JOBpostfix}"
1587 JOBID2wait="wm0_${JOBpostfix}"
1588 JOBID3="op0_${JOBpostfix}"
1589 JOBID3wait="wop0_${JOBpostfix}"
1590 JOBID4="p1_${JOBpostfix}"
1591 JOBID4wait="w1_${JOBpostfix}"
1592 JOBID5="m1_${JOBpostfix}"
1593 JOBID5wait="wm1_${JOBpostfix}"
1594 JOBID6="s1_${JOBpostfix}"
1595 JOBID6wait="ws1_${JOBpostfix}"
1596 JOBID7="QA_${JOBpostfix}"
1597 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1598 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1599 LASTJOB="000"
1600
71353a5f 1601 oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
cb473870 1602
1603 currentDefaultOCDB=${defaultOCDB}
1604 [[ -z ${autoOCDB} ]] && autoOCDB=1
1605 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1606 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1607 fi
1608
71353a5f 1609 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1610
1611 ###############################################################################
1612 #run one chunk with valgrind:
6fc48f9e 1613 if [[ -n ${profilingCommand} ]]; then
1614 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1615 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1616 submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
71353a5f 1617 fi
fc7e1e45 1618
5dfac970 1619 ################################################################################
1620 ################################################################################
1621 # run the CPass0 if requested
fc7e1e45 1622
71353a5f 1623 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1624
fc7e1e45 1625 echo
71353a5f 1626 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1627 echo
fc7e1e45 1628
5dfac970 1629 # create directory and copy all files that are needed
cb473870 1630 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1631 mkdir -p ${targetDirectory}
1632
cb473870 1633 filesCPass0=(
71353a5f 1634 "${configPath}/runCPass0.sh"
1635 "${configPath}/recCPass0.C"
1636 "${configPath}/runCalibTrain.C"
1637 "${configPath}/localOCDBaccessConfig.C"
1638 "${configPath}/OCDB*.root"
1639 "${configPath}/sim.C"
1640 "${configPath}/rec.C"
1641 "${configPath}/Config.C"
cb473870 1642 )
1643 for file in ${filesCPass0[*]}; do
1644 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1645 done
fc7e1e45 1646
71353a5f 1647 localInputList=${targetDirectory}/${inputList##*/}
1648 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1649 # limit nFiles to nMaxChunks
71353a5f 1650 nFiles=$(wc -l < ${localInputList})
1651 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1652 echo "raw files in list: ${nFiles}"
1653 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1654 nFiles=${nMaxChunks}
5dfac970 1655 fi
71353a5f 1656 echo "raw files to process: ${nFiles}"
1657 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1658 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1659 nFilesToWaitFor=${nFiles}
5dfac970 1660 else
71353a5f 1661 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1662 fi
71353a5f 1663 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1664 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1665
444b9916 1666 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1667
5dfac970 1668 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
444b9916 1669 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
71353a5f 1670 LASTJOB=${JOBID1wait}
fc7e1e45 1671
5dfac970 1672 fi #end running CPass0
1673 ################################################################################
fc7e1e45 1674
fc7e1e45 1675
5dfac970 1676 ################################################################################
1677 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1678
71353a5f 1679 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1680
5dfac970 1681 echo
71353a5f 1682 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1683 echo
fc7e1e45 1684
cb473870 1685 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
71353a5f 1686 mkdir -p ${targetDirectory}
fc7e1e45 1687
cb473870 1688 #copy the scripts
1689 filesMergeCPass0=(
71353a5f 1690 "${configPath}/OCDB.root"
1691 "${configPath}/mergeMakeOCDB.byComponent.sh"
1692 "${configPath}/mergeMakeOCDB.sh"
1693 "${configPath}/localOCDBaccessConfig.C"
1694 "${configPath}/mergeByComponent.C"
1695 "${configPath}/makeOCDB.C"
1696 "${configPath}/merge.C"
cb473870 1697 )
1698 for file in ${filesMergeCPass0[*]}; do
1699 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1700 done
1701
444b9916 1702 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1703 LASTJOB=${JOBID2}
fc7e1e45 1704
71353a5f 1705 if [[ -n ${generateMC} ]]; then
444b9916 1706 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1707 LASTJOB="mrl${JOBpostfix}"
1708 fi
fc7e1e45 1709
5dfac970 1710 echo
fc7e1e45 1711 fi
5dfac970 1712 # end of merging CPass0
1713 ################################################################################
fc7e1e45 1714
5dfac970 1715 ################################################################################
1716 ################################################################################
1717 # run the CPass1 if requested
fc7e1e45 1718
71353a5f 1719 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1720
cb473870 1721 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1722 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1723
5dfac970 1724 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1725 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1726
5dfac970 1727 echo
71353a5f 1728 echo "starting CPass1... for run ${runNumber}"
5dfac970 1729 echo
1730
1731 # create directory and copy all files that are needed
71353a5f 1732 mkdir -p ${targetDirectory}
5dfac970 1733
cb473870 1734 filesCPass1=(
71353a5f 1735 "${configPath}/runCPass1.sh"
1736 "${configPath}/recCPass1.C"
1737 "${configPath}/recCPass1_OuterDet.C"
1738 "${configPath}/runCalibTrain.C"
1739 "${configPath}/QAtrain_duo.C"
1740 "${configPath}/localOCDBaccessConfig.C"
1741 "${configPath}/OCDB.root"
cb473870 1742 )
1743 for file in ${filesCPass1[*]}; do
1744 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1745 done
5dfac970 1746
71353a5f 1747 if [[ -n ${generateMC} ]]; then
1748 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1749 else
1750 localInputList=${targetDirectory}/${inputList##*/}
1751 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1752 fi
5dfac970 1753 # limit nFiles to nMaxChunks
71353a5f 1754 nFiles=$(wc -l < ${localInputList})
1755 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1756 echo "raw files in list: ${nFiles}"
1757 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1758 nFiles=${nMaxChunks}
5dfac970 1759 fi
71353a5f 1760 echo "raw files to process: ${nFiles}"
1761 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1762 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1763 nFilesToWaitFor=${nFiles}
fc7e1e45 1764 else
71353a5f 1765 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 1766 fi
71353a5f 1767 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1768 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1769
444b9916 1770 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1771
5dfac970 1772 ################################################################################
1773 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
444b9916 1774 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
71353a5f 1775 LASTJOB=${JOBID4wait}
5dfac970 1776 ################################################################################
fc7e1e45 1777
5dfac970 1778 echo
1779 fi #end running CPass1
fc7e1e45 1780
5dfac970 1781 ################################################################################
1782 # submit merging of CPass1, depends on the reconstruction
71353a5f 1783 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1784
5dfac970 1785 echo
71353a5f 1786 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 1787 echo
fc7e1e45 1788
cb473870 1789 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
71353a5f 1790 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1791 mkdir -p ${targetDirectory}
fc7e1e45 1792
cb473870 1793 # copy files
1794 filesMergeCPass1=(
71353a5f 1795 "${configPath}/OCDB.root"
1796 "${configPath}/localOCDBaccessConfig.C"
1797 "${configPath}/mergeMakeOCDB.byComponent.sh"
1798 "${configPath}/mergeByComponent.C"
1799 "${configPath}/makeOCDB.C"
1800 "${configPath}/merge.C"
1801 "${configPath}/mergeMakeOCDB.sh"
1802 "${configPath}/QAtrain_duo.C"
cb473870 1803 )
1804 for file in ${filesMergeCPass1[*]}; do
1805 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1806 done
1807
444b9916 1808 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1809 LASTJOB=${JOBID5}
5dfac970 1810 echo
1811 fi
1812
444b9916 1813 ###############################
1814 #if [ ${runESDfiltering} -eq 1 ]; then
1815 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1816 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1817 # echo
1818 # echo submitting filtering for run ${runNumber}
1819 # echo
1820 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1821 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1822 # LASTJOB=${JOBfilterESDcpass1}
1823 #fi
cb473870 1824
5dfac970 1825 done
1826
71353a5f 1827 #################################################################################
1828 #################################################################################
444b9916 1829 #if [ ${runESDfiltering} -eq 1 ]; then
1830 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1831 #else
1832 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1833 #fi
71353a5f 1834 LASTJOB=${JOBID5wait}
cb473870 1835
5de3064b 1836 #################################################################################
1837 echo
1838 echo "submit make a summary"
1839 echo
5dfac970 1840
444b9916 1841 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
5de3064b 1842 LASTJOB=${JOBID6}
71353a5f 1843 #################################################################################
5dfac970 1844
5dfac970 1845 #restore stdout
1846 exec 1>&7 7>&-
1847 echo "jobs submitted."
5de3064b 1848 return 0
fc7e1e45 1849}
1850
cb473870 1851goWaitForOutput()
1852{
1853 umask 0002
71353a5f 1854 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1855 echo searchPath=${1}
1856 echo fileName=${2}
1857 echo numberOfFiles=${3}
1858 echo maxSecondsToWait=${4}
1859 searchPath=${1}
1860 fileName=${2}
1861 numberOfFiles=${3}
1862 maxSecondsToWait=${4}
1863 extraFindOptions=${5}
1864 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1865 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 1866 while true; do
71353a5f 1867 #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
cb473870 1868 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 1869 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1870 [[ ${n} -ge ${numberOfFiles} ]] && break
1871 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 1872 sleep 60
1873 done
1874 echo "DONE! exiting..."
5de3064b 1875 return 0
cb473870 1876}
1877
71353a5f 1878mergeSysLogs()
1879{
1880 outputFile=${1}
1881 shift
1882 inputFiles="$@"
1883 i=0
1884 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1885 while read x; do
1886 runNumber=$(guessRunNumber ${x})
1887 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1888 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1889 (( i++ ))
1890 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 1891 return 0
71353a5f 1892}
1893
1894goMakeMergedSummaryTree()
1895{
1896 # create list of calibration entries
1897 # takes no arguments, just run it in the base output
1898 # directory with the following files in the working directory
1899 #
1900 # Calibration file lists:
1901 # cpass0.dcsTree.list, cpass1.dcsTree.list
1902 # QA trending root file:
1903 # trending.root
1904 #
1905 # Production infoo ascii files:
1906 # summary_pass0.tree
1907 # summary_pass1.tree
1908 #
617f8886 1909
71353a5f 1910 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1911 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1912 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1913 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1914 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1915
1916 #first, dump the C macro to file
1917 cat << EOF > mergeTree.C
1918 //
1919 // Merge summary information
1920 // Following files are expected to be in the working directory
1921 //
1922 // Calibration file lists:
1923 // cpass0.dcsTree.list, cpass1.dcsTree.list
1924 // QA trending root file:
1925 // trending.root
1926 //
1927 // Production infoo ascii files:
1928 // summary_pass0.tree
1929 // summary_pass1.tree
1930 //
1931 void mergeTree(){
1932 //
1933 //
1934 //
1935 // Calibration values dump
1936 //
64a8d0f6 1937 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 1938 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
64a8d0f6 1939 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 1940 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1941 //
1942 // Calibration status dump
1943 //
1944 TFile *fprod = TFile::Open("fproduction.root","recreate");
1945 TTree tree0, tree1;
64a8d0f6 1946 //Printf("reading summary_pass0.tree");
71353a5f 1947 tree0.ReadFile("summary_pass0.tree");
64a8d0f6 1948 //Printf("reading summary_pass1.tree");
71353a5f 1949 tree1.ReadFile("summary_pass1.tree");
1950 tree0.Write("CPass0");
1951 tree1.Write("CPass1");
1952 fprod->Close();
1953 //
1954 //
1955 //
1956 TString stringSetup="";
1957 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
1958 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
1959 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
1960 //
1961 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
1962 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
1963 //
64a8d0f6 1964 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 1965 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1966 }
1967EOF
1968
1969 aliroot -b -q "mergeTree.C" > mergeTrees.log
1970 return $?
1971}
1972
1973goMakeSummary()
1974{
1975 #all the final stuff goes in here for ease of use:
1976 # summary logs
1977 # qa plot making
1978 # final file lists
444b9916 1979 #some defaults:
1980 log="summary.log"
1981 productionID="qa"
71353a5f 1982
1983 configFile=${1}
1984 shift 1
444b9916 1985 extraOpts=("$@")
6fc48f9e 1986 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 1987
64a8d0f6 1988 configFile=$(readlink -f ${configFile})
1989
39527a15 1990 #record the working directory provided by the batch system
1991 batchWorkingDirectory=${PWD}
1992
71353a5f 1993 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1994
71353a5f 1995 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1996
1997 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1998
39527a15 1999 #copy some useful stuff
2000 #and go to the commonOutputPath
2001 cp ${configFile} ${commonOutputPath}
2002 cd ${commonOutputPath}
2003
2004 exec &> >(tee ${log})
2005
71353a5f 2006 #summarize the global stuff
2007 echo "env script: ${alirootSource} ${alirootEnv}"
2008 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2009 echo "commonOutputPath=${commonOutputPath}"
2010
2011 #summarize the stacktraces
2012 awk '
2013 BEGIN {
2014 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2015 RS="#[0-9]*";
2016 aliroot=0;
2017 }
2018 {
2019 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2020 gsub("#","",RT);
2021 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2022 }
2023 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2024 awk '
2025 BEGIN {
2026 RS="#[0-9]*";
2027 aliroot=0;
2028 }
2029 {
2030 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2031 gsub("#","",RT);
2032 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2033 }
2034 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2035
2036 echo total numbers for the production:
2037 echo
2038 awk 'BEGIN {nFiles=0;nCore=0;}
2039 /^calibfile/ {nFiles++;}
2040 /core dumped/ {nCore++i;}
2041 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2042 awk 'BEGIN {nOK=0; nBAD=0; }
2043 /\/rec.log OK/ {nOK++;}
2044 /\/rec.log BAD/ {nBAD++;}
2045 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2046 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2047 awk 'BEGIN {nOK=0; nBAD=0; }
2048 /\/calib.log OK/ {nOK++;}
2049 /\/calib.log BAD/ {nBAD++;}
2050 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2051
2052 awk 'BEGIN {nOK=0; nBAD=0; }
2053 /merge.log OK/ {nOK++;}
2054 /merge.log BAD/ {nBAD++;}
2055 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2056 awk 'BEGIN {nOK=0; nBAD=0; }
2057 /ocdb.log OK/ {nOK++;}
2058 /ocdb.log BAD/ {nBAD++;}
2059 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2060
2061 echo
2062 awk 'BEGIN {nFiles=0;nCore=0;}
2063 /^calibfile/ {nFiles++;}
2064 /core dumped/ {nCore++;}
2065 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2066 awk 'BEGIN {nOK=0; nBAD=0; }
2067 /\/rec.log OK/ {nOK++;}
2068 /\/rec.log BAD/ {nBAD++;}
2069 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2070 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2071 awk 'BEGIN {nOK=0; nBAD=0; }
2072 /\/calib.log OK/ {nOK++;}
2073 /\/calib.log BAD/ {nBAD++;}
2074 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2075
2076 awk 'BEGIN {nOK=0; nBAD=0; }
2077 /merge.log OK/ {nOK++;}
2078 /merge.log BAD/ {nBAD++;}
2079 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2080 awk 'BEGIN {nOK=0; nBAD=0; }
2081 /ocdb.log OK/ {nOK++;}
2082 /ocdb.log BAD/ {nBAD++;}
2083 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2084
2085 echo
2086 echo per run stats:
2087 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2088do
2089 dir=$(goPrintValues calibfile - ${x})
2090 runNumber=$(guessRunNumber ${dir})
2091 [[ -z ${runNumber} ]] && continue
2092
2093 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2094 statusCPass0=( $(
2095 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2096 /\/rec.log OK/ {nOKrec++;}
2097 /\/rec.log BAD/ {nBADrec++;}
2098 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2099 /stderr OK/ {nOKstderr++;}
2100 /\/calib.log OK/ {nOKcalib++;}
2101 /\/calib.log BAD/ {nBADcalib++}
2102 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2103 ) )
2104 fi
2105
2106 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2107 statusCPass1=( $(
2108 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2109 /\/rec.log OK/ {nOKrec++;}
2110 /\/rec.log BAD/ {nBADrec++;}
2111 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2112 /stderr OK/ {nOKstderr++;}
2113 /\/calib.log OK/ {nOKcalib++;}
2114 /\/calib.log BAD/ {nBADcalib++}
2115 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2116 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2117 /\/qa_outer.log OK/ {nQAouterOK++;}
2118 /\/qa_outer.log BAD/ {nQAouterBAD++;}
2119 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2120 ) )
2121 fi
2122
2123 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2124 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2125 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2126
2127 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2128done
2129
2130 #make lists with output files - QA, trending, filtering and calibration
2131 ### wait for the merging of all runs to be over ###
2132 rm -f qa.list
2133 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2134 rm -f calib.list
2135 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2136 rm -f trending.list
2137 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2138 rm -f filtering.list
444b9916 2139 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2140 rm -f cpass0.dcsTree.list
2141 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2142 rm -f cpass1.dcsTree.list
2143 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2144
2145 #merge trending
2146 rm -f ${commonOutputPath}/trending_merged.root
444b9916 2147 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2148
2149 goMakeSummaryTree ${commonOutputPath} 0
2150 goMakeSummaryTree ${commonOutputPath} 1
2151
64a8d0f6 2152 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2153
2154 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
64a8d0f6 2155 echo goMakeMergedSummaryTree PWD=$PWD
71353a5f 2156 goMakeMergedSummaryTree
2157
2158 #if set, email the summary
5de3064b 2159 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
71353a5f 2160
2161 return 0
2162}
2163
2164goMakeSummaryTree()
2165{
2166 if [[ $# -lt 1 ]] ; then
2167 return
2168 fi
2169 #1. define vars/arrays
2170 DIR=${1} #use input or exec in current dir
2171 pass=${2-"0"} #pass from input
2172 outfile="summary_pass${pass}.tree"
2173 Ncolumns=0
2174 test -f ${outfile} && : >${outfile}
2175 errfile=${outfile/tree/err}
2176 test -f ${errfile} && : >${errfile}
2177
2178 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2179 Ncounter=${#counterString[@]}
2180
2181 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2182 Nstatus=${#statusString[@]}
2183
2184
2185 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2186 Nrates=${#ratesString[@]}
2187
2188 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2189 Nruns=${#runs[@]}
2190
2191 echo -n runnumber/I >>${outfile}
2192 echo -n :cpass${pass}status/I >>${outfile}
2193 echo -n :cpass${pass}QAstatus/I >>${outfile}
2194 for i in ${ratesString[@]}; do
2195 echo -n :${i}OK/I >>${outfile}
2196 echo -n :${i}BAD/I >>${outfile}
2197
2198 done
2199 for i in ${counterString[@]} ${statusString[@]} ; do
2200 echo -n :${i}/I >>${outfile}
2201 done
2202 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2203 echo >> ${outfile}
2204
2205 #2. loop runs
2206
2207 for runnumber in ${runs[@]} ; do
2208
2209
2210
2211 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2212 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2213 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2214 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2215 echo "${filemerge} does not exist!" >>${errfile}
2216 continue
2217 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2218 echo "${filejob} does not exist!" >>${errfile}
2219 echo -n ${runnumber} >> ${outfile}
2220 for i in $(seq ${Ncolumns}) ; do
2221 echo -n "-1" >> ${outfile}
2222 done
2223 echo >> ${outfile}
2224 continue
2225 fi
2226 echo -n ${runnumber} >> ${outfile}
2227 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2228 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2229 echo -n " ${passStatus}" >> ${outfile}
2230 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2231 echo -n " ${qaStatus}" >> ${outfile}
2232
2233
2234 #fill OK/BAD rates
2235 for i in $(seq 0 $((${Nrates}-1))) ; do
2236 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2237 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2238
2239 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2240 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2241 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2242 fi
2243 echo -n " ${var1}" >> ${outfile}
2244 echo -n " ${var2}" >> ${outfile}
2245 done
2246
2247 if [[ -f ${fileOCDB} ]] ; then
2248 #fill counter
2249 for i in $(seq 0 $((${Ncounter}-1))) ; do
2250 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2251 echo -n " ${var1:-"-1"}" >> ${outfile}
2252 done
2253
2254 #fill status
2255 for i in $(seq 0 $((${Nstatus}-1))) ; do
2256 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2257 echo -n " ${var1:-"-1"}" >> ${outfile}
2258 done
2259 fi
2260 echo >> ${outfile}
2261 done
2262
5de3064b 2263 return 0
71353a5f 2264}
2265
2266parseConfig()
2267{
2268 configFile=${1}
2269 shift
444b9916 2270 args=("$@")
2271
2272 #some defaults
6fc48f9e 2273 #autoOCDB=0
2274 defaultOCDB="raw://"
2275 #runNumber=167123
2276 #makeflowPath="/hera/alice/aux/cctools/bin"
2277 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2278 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2279 makeflowOptions=""
39527a15 2280 #batchCommand="/usr/bin/qsub"
6fc48f9e 2281 batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2282 baseOutputDirectory="$PWD/output"
2283 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2284 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2285 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2286 reconstructInTemporaryDir=0
2287 recoTriggerOptions="\"\""
2288 percentProcessedFilesToContinue=100
2289 maxSecondsToWait=$(( 3600*24 ))
2290 nEvents=-1
2291 nMaxChunks=0
2292 postSetUpActionCPass0=""
2293 postSetUpActionCPass1=""
2294 runCPass0reco=1
2295 runCPass0MergeMakeOCDB=1
2296 runCPass1reco=1
2297 runCPass1MergeMakeOCDB=1
2298 runESDfiltering=1
444b9916 2299 filteringFactorHighPt=1e2
2300 filteringFactorV0s=1e1
6fc48f9e 2301 MAILTO=""
2302 #pretend=1
2303 #dontRedirectStdOutToLog=1
2304 logToFinalDestination=1
2305 ALIROOT_FORCE_COREDUMP=1
71353a5f 2306
2307 #first, source the config file
2308 if [ -f ${configFile} ]; then
2309 source ${configFile}
2310 else
2311 echo "config file ${configFile} not found!, skipping..."
2312 fi
2313
64a8d0f6 2314 unset encodedSpaces
2315 for opt in "${args[@]}"; do
2316 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && break
2317 done
2318
d56eeaab 2319 #then, parse the options as they override the options from file
2320 for opt in "${args[@]}"; do
2321 if [[ ! "${opt}" =~ .*=.* ]]; then
2322 echo "badly formatted option ${var}, should be: option=value, stopping..."
2323 return 1
2324 fi
2325 local var="${opt%%=*}"
2326 local value="${opt#*=}"
64a8d0f6 2327 [[ -n ${encodedSpaces} ]] && value=$(decSpaces "${value}")
d56eeaab 2328 echo "${var} = ${value}"
444b9916 2329 export ${var}="${value}"
71353a5f 2330 done
2331
6fc48f9e 2332 #do some checking
2333 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2334
71353a5f 2335 #export the aliroot function if defined to override normal behaviour
2336 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
6fc48f9e 2337
5de3064b 2338 return 0
71353a5f 2339}
2340
2341aliroot()
2342{
2343 args="$@"
6fc48f9e 2344 if [[ -n ${useProfilingCommand} ]]; then
71353a5f 2345 valgrindLogFile="cpu.txt"
2346 [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2347 [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
6fc48f9e 2348 [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2349 echo running ${useProfilingCommand} aliroot ${args}
2350 ${useProfilingCommand} aliroot ${args}
71353a5f 2351 else
2352 #to prevent an infinite recursion use "command aliroot" to disable
2353 #aliases and functions
2354 echo running command aliroot ${args}
444b9916 2355 command aliroot "$@"
71353a5f 2356 fi
5de3064b 2357 return 0
71353a5f 2358}
2359
2360guessRunData()
2361{
2362 #guess the period from the path, pick the rightmost one
2363 period=""
2364 runNumber=""
2365 year=""
2366 pass=""
2367 legoTrainRunNumber=""
2368 dataType=""
2369
2370 local shortRunNumber=""
2371 local IFS="/"
2372 declare -a path=( $1 )
2373 local dirDepth=$(( ${#path[*]}-1 ))
2374 i=0
2375 #for ((x=${dirDepth};x>=0;x--)); do
2376 for ((x=0;x<=${dirDepth};x++)); do
2377
2378 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2379 local field=${path[${x}]}
2380 local fieldNext=${path[$((x+1))]}
2381
2382 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2383 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2384 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2385 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2386 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2387 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2388 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2389 (( i++ ))
2390 done
2391 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2392 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2393
2394 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2395 if [[ -z ${runNumber}} ]];
2396 then
2397 #error condition
2398 return 1
2399 else
2400 #ALL OK
2401 return 0
2402 fi
5de3064b 2403 return 0
71353a5f 2404}
2405
64a8d0f6 2406#these functions encode strings to and from a space-less form
2407#use when spaces are not well handled (e.g. in arguments to
2408#commands in makeflow files, etc.
2409encSpaces()(a="${1//,/\\,}";echo "${a// /,}")
2410decSpaces()(a="${1//\\,/\\ }";b="${a//,/ }";echo "${b//\\ /,}")
2411
fc7e1e45 2412main "$@"