]> git.uio.no Git - u/mrichter/AliRoot.git/blame_incremental - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
some fixes for running with makeflow
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
... / ...
CommitLineData
1#!/bin/bash
2#include benchmark.config
3
4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
17 echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10"
18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
24 umask 0002
25 shift
26 case ${runMode} in
27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33 "MakeSummary") goMakeSummary "$@";;
34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
36 "test") goTest "$@";;
37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
41 "Merge") goMerge "$@";;
42 *)
43 eval ${runMode} $@
44 ;;
45 esac
46}
47
48generateMC()
49{
50 #generate one raw chunk in current directory
51 SEED=${JOB_ID}${SGE_TASK_ID}
52 export CONFIG_SEED=${SEED}
53 runNumber=${1}
54 OCDBpath=${2}
55 nEventsim=${3}
56 if [[ -n ${pretend} ]]; then
57 touch galice.root
58 else
59 if [[ -f sim.C && -f Config.C ]] ; then
60 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
61 mv syswatch.log simwatch.log
62 fi
63 fi
64}
65
66goCPass0()
67{
68 umask 0002
69
70 targetDirectory=${1}
71 inputList=${2}
72 nEvents=${3}
73 ocdbPath=${4}
74 configFile=${5}
75 runNumber=${6}
76 jobindex=${7}
77 shift 7
78 parseConfig ${configFile} $@
79
80 #use the jobindex only if set and non-negative
81 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
82 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
83 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
84 fi
85
86 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
87 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
88
89 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
90
91 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
92 ulimit -c unlimited
93 export ALIROOT_FORCE_COREDUMP
94 fi
95
96 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
97 #at the QA stage the pass number is guessed from the path stored here.
98 #The Format is:
99 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
100 export PRODUCTION_METADATA="OutputDir=cpass0"
101
102 if [[ "${inputList}" =~ \.root$ ]]; then
103 infile=${inputList}
104 else
105 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
106 fi
107 chunkName=${infile##*/}
108
109 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
110 mkdir -p ${outputDir}
111 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
112
113 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
114 runpath=${outputDir}
115 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
116 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
117 mkdir -p ${runpath}
118 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
119 cd ${runpath}
120
121 #runCPassX/C expects the raw chunk to be linked in the run dir
122 #despite it being accessed by the full path
123 ln -s ${infile} ${runpath}/${chunkName}
124
125 #####MC
126 if [[ -n ${generateMC} ]]; then
127 olddir=${PWD}
128 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
129 simrunpath=${outputDirMC}
130 [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
131 [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
132 mkdir -p ${outputDirMC}
133 mkdir -p ${simrunpath}
134 cd ${simrunpath}
135
136 filesMC=(
137 "${commonOutputPath}/sim.C"
138 "${commonOutputPath}/rec.C"
139 "${commonOutputPath}/Config.C"
140 "${commonOutputPath}/OCDB_*.root"
141 )
142 for file in ${filesMC[*]}; do
143 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
144 done
145
146 generateMC ${runNumber} ${ocdbPath} ${nEvents}
147
148 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
149 cd ${olddir}
150
151 ln -s ${outputDirMC}/* ${runpath}/
152
153 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
154 infile=""
155 fi
156 ######
157
158 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
159
160 logOutputDir=${runpath}
161 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
162 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
163 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
164 echo "${0} $*"
165
166 echo "#####################"
167 echo CPass0:
168 echo JOB setup
169 echo nEvents ${nEvents}
170 echo runNumber ${runNumber}
171 echo ocdbPath ${ocdbPath}
172 echo infile ${infile}
173 echo chunkName ${chunkName}
174 echo jobindex ${jobindex}
175 echo recoTriggerOptions ${recoTriggerOptions}
176 echo targetDirectory ${targetDirectory}
177 echo commonOutputPath ${commonOutputPath}
178 echo doneFile ${doneFile}
179 echo runpath ${runpath}
180 echo outputDir ${outputDir}
181 echo ALICE_ROOT ${ALICE_ROOT}
182 echo PWD ${PWD}
183 echo "########## ###########"
184
185 alirootInfo > ALICE_ROOT.log
186
187 filesCPass0=(
188 "${commonOutputPath}/runCPass0.sh"
189 "${commonOutputPath}/recCPass0.C"
190 "${commonOutputPath}/runCalibTrain.C"
191 "${commonOutputPath}/localOCDBaccessConfig.C"
192 "${commonOutputPath}/OCDB.root"
193 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
194 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
195 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
196 )
197
198 for file in ${filesCPass0[*]}; do
199 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
200 done
201
202 echo "this directory (${PWD}) contents:"
203 /bin/ls
204 echo
205 chmod u+x runCPass0.sh
206
207 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
208
209 if [[ -n ${postSetUpActionCPass0} ]]; then
210 echo "running ${postSetUpActionCPass0}"
211 eval ${postSetUpActionCPass0}
212 fi
213
214 #run CPass0
215 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
216 if [[ -n ${pretend} ]]; then
217 touch AliESDs.root
218 touch AliESDfriends.root
219 touch AliESDfriends_v1.root
220 touch rec.log
221 touch calib.log
222 else
223 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
224 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
225 fi
226
227 #move stuff to final destination
228 echo "this directory (${PWD}) contents:"
229 /bin/ls
230 echo
231
232 echo rm -f ./${chunkName}
233 rm -f ./${chunkName}
234 echo "cp --recursive ${runpath}/* ${outputDir}"
235 cp -p --recursive ${runpath}/* ${outputDir}
236 echo
237
238 #validate CPass0
239 cd ${outputDir}
240 touch ${doneFile}
241 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
242 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
243 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
244 echo "dir ${outputDir}" >> ${doneFile}
245 summarizeLogs >> ${doneFile}
246
247 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
248 return 0
249}
250
251goCPass1()
252{
253 umask 0002
254
255 targetDirectory=${1}
256 inputList=${2}
257 nEvents=${3}
258 ocdbPath=${4}
259 configFile=${5}
260 runNumber=${6}
261 jobindex=${7}
262 shift 7
263 parseConfig ${configFile} $@
264
265 #use the jobindex only if set and non-negative
266 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
267 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
268 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
269 fi
270
271 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
272 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
273
274 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
275
276 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
277 ulimit -c unlimited
278 export ALIROOT_FORCE_COREDUMP
279 fi
280
281 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
282 #at the QA stage the pass number is guessed from the path stored here.
283 #The Format is:
284 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
285 export PRODUCTION_METADATA="OutputDir=cpass1"
286
287 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
288 if [[ "${inputList}" =~ \.root$ ]]; then
289 infile=${inputList}
290 else
291 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
292 fi
293 chunkName=${infile##*/}
294
295 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
296 mkdir -p ${outputDir}
297 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
298
299 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
300 runpath=${outputDir}
301 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
302 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
303
304 #MC
305 if [[ "${infile}" =~ galice\.root ]]; then
306 ln -s ${inputList%/*}/* ${runpath}
307 infile=""
308 fi
309
310 #init the running path
311 mkdir -p ${runpath}
312 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
313 cd ${runpath}
314
315 #this is needed for runCPass1.sh
316 ln -s ${infile} ${runpath}/${chunkName}
317
318 logOutputDir=${runpath}
319 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
320 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
321 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
322 echo "${0} $*"
323
324 echo "#####################"
325 echo CPass1:
326 echo JOB setup
327 echo nEvents ${nEvents}
328 echo runNumber ${runNumber}
329 echo ocdbPath ${ocdbPath}
330 echo infile ${infile}
331 echo chunkName ${chunkName}
332 echo jobindex ${jobindex}
333 echo recoTriggerOptions ${recoTriggerOptions}
334 echo targetDirectory ${targetDirectory}
335 echo commonOutputPath ${commonOutputPath}
336 echo doneFile ${doneFile}
337 echo runpath ${runpath}
338 echo outputDir ${outputDir}
339 echo ALICE_ROOT ${ALICE_ROOT}
340 echo PWD ${PWD}
341 echo "########## ###########"
342
343 alirootInfo > ALICE_ROOT.log
344
345 filesCPass1=(
346 "${commonOutputPath}/runCPass1.sh"
347 "${commonOutputPath}/recCPass1.C"
348 "${commonOutputPath}/recCPass1_OuterDet.C"
349 "${commonOutputPath}/runCalibTrain.C"
350 "${commonOutputPath}/QAtrain_duo.C"
351 "${commonOutputPath}/localOCDBaccessConfig.C"
352 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
353 "${commonOutputPath}/OCDB.root"
354 "${trustedQAtrainMacro}"
355 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
356 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
357 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
358 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
359 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
360 )
361
362 for file in ${filesCPass1[*]}; do
363 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
364 done
365
366 echo "this directory (${PWD}) contents:"
367 /bin/ls
368 echo
369
370 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
371
372 if [[ -n ${postSetUpActionCPass1} ]]; then
373 echo "running ${postSetUpActionCPass1}"
374 eval ${postSetUpActionCPass1}
375 echo
376 fi
377
378 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
379 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
380 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
381 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
382 else
383 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
384 fi
385
386 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
387 echo "cpass0 produced no calibration! exiting..."
388 touch ${doneFile}
389 return 1
390 fi
391
392 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
393 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
394 #entries using a relative path, e.g. local://./OCDB
395 mkdir Barrel OuterDet
396 ln -s ../OCDB Barrel/OCDB
397 ln -s ../OCDB OuterDet/OCDB
398
399 #run CPass1
400 chmod u+x runCPass1.sh
401 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
402 if [[ -n ${pretend} ]]; then
403 touch AliESDs_Barrel.root
404 touch AliESDfriends_Barrel.root
405 touch AliESDfriends_v1.root
406 touch QAresults_barrel.root
407 touch EventStat_temp_barrel.root
408 touch AODtpITS.root
409 touch AliESDs_Outer.root
410 touch AliESDfriends_Outer.root
411 touch QAresults_outer.root
412 touch EventStat_temp_outer.root
413 touch rec.log
414 touch calib.log
415 touch qa.log
416 else
417 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
418 fi
419
420 ##handle possible crashes in QA (happens often in trunk)
421 ##rerun QA with a trusted aliroot version
422 #if [[ $(validateLog qa_barrel.log) ]]; then
423 # echo "qa_barrel.log not validated!"
424 #fi
425 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
426 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
427 # source ${setupTrustedAliROOTenvInCurrentShell}
428 # cd Barrel
429 # rm QAresults_barrel.root
430 # rm EventStat_temp_barrel.root
431 # rm AODtpITS.root
432 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
433 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
434 # if [[ -n ${trustedQAtrainMacro} ]]; then
435 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
436 # fi
437 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
438 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
439 # cd ../
440 #fi
441 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
442 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
443 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
444 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
445 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
446 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
447 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
448
449 #move stuff to final destination
450 echo "this directory (${PWD}) contents:"
451 /bin/ls
452 echo rm -f ./${chunkName}
453 rm -f ./${chunkName}
454 echo "cp --recursive ${runpath}/* ${outputDir}"
455 cp -pf --recursive ${runpath}/* ${outputDir}
456 echo
457
458 #validate CPass1
459 cd ${outputDir}
460 touch ${doneFile}
461 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
462 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
463 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
464 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
465 echo "dir ${outputDir}" >> ${doneFile}
466 summarizeLogs >> ${doneFile}
467
468 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
469 return 0
470}
471
472
473goMergeCPass0()
474{
475 #
476 # find the output files and merge them
477 #
478
479 outputDir=${1}
480 ocdbStorage=${2}
481 configFile=${3}
482 runNumber=${4}
483 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
484 shift 5
485 parseConfig ${configFile} $@
486
487 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
488 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
489
490 umask 0002
491 ulimit -c unlimited
492
493 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
494
495 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
496 runpath=${outputDir}
497 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
498 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
499
500 mkdir -p ${runpath}
501 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
502 cd ${runpath}
503
504 logOutputDir=${runpath}
505 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
506 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
507 echo "${0} $*"
508
509 mergingScript="mergeMakeOCDB.byComponent.sh"
510
511 echo goMergeCPass0 SETUP:
512 echo runNumber=${runNumber}
513 echo outputDir=${outputDir}
514 echo ocdbStorage=${ocdbStorage}
515 echo calibrationFilesToMerge=${calibrationFilesToMerge}
516 echo mergingScript=${mergingScript}
517 echo commonOutputPath=${commonOutputPath}
518 echo runpath=${runpath}
519
520 # copy files in case they are not already there
521 filesMergeCPass0=(
522 "${commonOutputPath}/${calibrationFilesToMerge}"
523 "${commonOutputPath}/OCDB.root"
524 "${commonOutputPath}/localOCDBaccessConfig.C"
525 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
526 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
527 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
528 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
529 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
530 )
531 for file in ${filesMergeCPass0[*]}; do
532 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
533 done
534
535 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
536
537 alirootInfo > ALICE_ROOT.log
538
539 #
540 echo "PWD"
541 /bin/ls
542 echo "PWD/.."
543 /bin/ls ../
544
545
546 #merge calibration
547 chmod u+x ${mergingScript}
548 mkdir -p ./OCDB
549 if [[ ! -f ${calibrationFilesToMerge} ]]; then
550 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
551 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
552 fi
553
554 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
555 if [[ -n ${pretend} ]]; then
556 touch CalibObjects.root
557 touch ocdb.log
558 touch merge.log
559 mkdir -p ./OCDB/TPC/Calib/TimeGain/
560 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
561 touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
562 touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
563 else
564 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
565 fi
566
567 ### produce the output
568 #tar the produced OCDB for reuse
569 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
570
571 #produce the calib trees for expert QA (dcsTime.root)
572 goMakeLocalOCDBaccessConfig ./OCDB
573 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
574 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
575
576 /bin/ls
577
578 #copy all to output dir
579 cp -pf --recursive ${runpath}/* ${outputDir}
580
581 if [[ -n ${generateMC} ]]; then
582 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
583 fi
584
585 #validate merging cpass0
586 cd ${outputDir}
587 touch ${doneFile}
588 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
589 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
590 echo "dir ${outputDir}" >> ${doneFile}
591 summarizeLogs >> ${doneFile}
592
593 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
594 return 0
595}
596
597goMergeCPass1()
598{
599 #
600 # find the output files and merge them
601 #
602
603 outputDir=${1}
604 ocdbStorage=${2}
605 configFile=${3}
606 runNumber=${4}
607 calibrationFilesToMerge=${5}
608 qaFilesToMerge=${6}
609 shift 6
610 parseConfig ${configFile} $@
611
612 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
613 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
614
615 umask 0002
616 ulimit -c unlimited
617
618 #clean up first:
619 rm -f ${outputDir}/*.log
620 rm -f ${outputDir}/*.root
621 rm -f ${outputDir}/*done
622
623 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
624
625 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
626 runpath=${outputDir}
627 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
628 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
629
630 mkdir -p ${runpath}
631 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
632 cd ${runpath}
633
634 logOutputDir=${runpath}
635 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
636 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
637 echo "${0} $*"
638
639 calibrationOutputFileName='AliESDfriends_v1.root'
640 qaOutputFileName='QAresults*.root'
641 mergingScript="mergeMakeOCDB.byComponent.sh"
642 #important to have the string "Stage.txt" in the filename to trigger the merging
643 #it has to be a list of directories containing the files
644 qaMergedOutputFileName="QAresults_merged.root"
645
646 echo goMergeCPass1 SETUP:
647 echo runNumber=${runNumber}
648 echo outputDir=${outputDir}
649 echo ocdbStorage=${ocdbStorage}
650 echo calibrationFilesToMerge=$calibrationFilesToMerge
651 echo qaFilesToMerge=$qaFilesToMerge
652 echo calibrationOutputFileName=${calibrationOutputFileName}
653 echo mergingScript=${mergingScript}
654
655 # copy files in case they are not already there
656 filesMergeCPass1=(
657 "${commonOutputPath}/${calibrationFilesToMerge}"
658 "${commonOutputPath}/${qaFilesToMerge}"
659 "${commonOutputPath}/OCDB.root"
660 "${commonOutputPath}/localOCDBaccessConfig.C"
661 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
662 "${commonOutputPath}/QAtrain_duo.C"
663 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
664 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
665 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
666 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
667 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
668 "${trustedQAtrainMacro}"
669 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
670 )
671 for file in ${filesMergeCPass1[*]}; do
672 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
673 done
674
675 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
676
677 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
678 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
679 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
680 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
681 else
682 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
683 fi
684
685 alirootInfo > ALICE_ROOT.log
686
687 #
688 /bin/ls
689
690 #merge calibration
691 chmod u+x ${mergingScript}
692 mkdir -p OCDB
693 if [[ ! -f ${calibrationFilesToMerge} ]]; then
694 echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}"
695 /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge}
696 fi
697
698 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
699 if [[ -n ${pretend} ]]; then
700 touch CalibObjects.root
701 touch ocdb.log
702 touch merge.log
703 touch cpass1.localOCDB.${runNumber}.tgz
704 else
705 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
706 fi
707
708 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
709
710 #merge QA
711 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
712 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
713
714 if [[ ! -f ${qaFilesToMerge} ]]; then
715 #find the files, but only store the directories (QAtrain_duo.C requires this)
716 echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
717 eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
718 fi
719
720 #do the merge
721 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
722 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
723 if [[ -n ${pretend} ]]; then
724 touch ${qaMergedOutputFileName}
725 touch merge.log
726 touch trending.root
727 else
728 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
729 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
730 mv QAresults_barrel.root ${qaMergedOutputFileName}
731 mv trending_barrel.root trending.root
732 fi
733
734 #produce the calib trees for expert QA
735 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
736 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
737
738 /bin/ls
739
740 #copy all to output dir
741 cp -pf --recursive ${runpath}/* ${outputDir}
742
743 #validate merge cpass1
744 cd ${outputDir}
745 touch ${doneFile}
746 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
747 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
748 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
749 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
750 echo "dir ${outputDir}" >> ${doneFile}
751 summarizeLogs >> ${doneFile}
752
753 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
754 return 0
755}
756
757goMerge()
758{
759 #generic root merge using CPass1 merge.C script
760 inputList=${1}
761 outputFile=${2}
762 configFile=${3-"becnhmark.config"}
763 shift 3
764 parseConfig ${configFile} $@
765
766 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
767 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
768 umask 0002
769 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
770 rm -f ${outputFile}
771 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
772 return 0
773}
774
775goSubmitMakeflow()
776{
777 #run
778 productionID=${1}
779 inputFileList=${2}
780 configFile=${3}
781 shift 3
782 extraOpts="$@"
783 parseConfig ${configFile} ${extraOpts}
784
785 [[ -z ${configFile} ]] && configFile="benchmark.config"
786 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
787
788 #create the directopry for the metadata
789 mkdir meta
790
791 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
792 echo "setting the makflow path from the config: "
793 echo " export PATH=${makeflowPath}:${PATH}"
794 export PATH=${makeflowPath}:${PATH}
795 fi
796
797 #submit - use makeflow if available, fall back to old stuff when makeflow not there
798 if which makeflow; then
799
800 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow
801
802 makeflow ${makeflowOptions} benchmark.makeflow
803 cd ../
804 else
805 echo "no makeflow!"
806 fi
807 return 0
808}
809
810goGenerateMakeflow()
811{
812 #generate the makeflow file
813 productionID=${1}
814 inputFileList=${2}
815 configFile=${3}
816 shift 3
817 extraOpts="$@"
818 parseConfig ${configFile} ${extraOpts}
819
820 [[ -z ${configFile} ]] && configFile="benchmark.config"
821 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
822
823 commonOutputPath=${baseOutputDirectory}/${productionID}
824
825 #these files will be made a dependency - will be copied to the working dir of the jobs
826 declare -a copyFiles
827 inputFiles=(
828 "OCDB.root"
829 "localOCDBaccessConfig.C"
830 "QAtrain_duo.C"
831 "runCPass1.sh"
832 "recCPass1.C"
833 "recCPass1_OuterDet.C"
834 "runCalibTrain.C"
835 "runCPass0.sh"
836 "recCPass0.C"
837 )
838 for file in ${inputFiles[*]}; do
839 [[ -f ${file} ]] && copyFiles+=("${file}")
840 done
841
842 #create the makeflow file
843 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
844 declare -a arr_cpass1_final
845 listOfRuns[0]=${runNumber}
846 [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq))
847 runindex=0
848 for runNumber in ${listOfRuns[*]}; do
849 [[ -z ${runNumber} ]] && continue
850 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
851
852 unset arr_cpass0_outputs_onerun
853 unset arr_cpass1_outputs_onerun
854 declare -a arr_cpass0_outputs_onerun
855 declare -a arr_cpass1_outputs_onerun
856
857 jobindex=0
858 while read inputFile; do
859 currentDefaultOCDB=${defaultOCDB}
860 [[ -z ${autoOCDB} ]] && autoOCDB=1
861 if [[ ${autoOCDB} -ne 0 ]]; then
862 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
863 fi
864
865 #CPass0
866 arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
867 echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
868 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
869 echo
870
871 #CPass1
872 arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
873 echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
874 echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
875 echo
876 ((jobindex++))
877
878 done< <(grep "/000${runNumber}/" ${inputFileList})
879
880 #CPass0 list of Calib files to merge
881 arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list"
882 echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
883 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}"
884 echo
885
886 #CPass0 merging
887 arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done"
888 echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}"
889 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}"
890 echo
891
892 #CPass1 list of Calib/QA files to merge
893 # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
894 # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
895 # this corresponds to the last merging stage)
896 arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list"
897 echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
898 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};"
899 echo
900 arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
901 echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
902 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
903 echo
904
905 #CPass1 merging
906 arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done"
907 echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}"
908 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}"
909 echo
910
911 arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list"
912 echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
913 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
914
915 if [ ${runESDfiltering} -eq 1 ]; then
916 arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done"
917 echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}"
918 echo " ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
919 fi
920
921 ((runindex++))
922 done
923
924 #Summary
925 echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}"
926 echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
927 echo
928
929 return 0
930}
931
932goPrintValues()
933{
934 #print the values given the key from any number of files (store in output file on second argument)
935 if [[ $# -lt 3 ]]; then
936 echo "goPrintValues key outputFile inputFiles"
937 echo "if outputFile is \"-\" dont redirect to a file"
938 return
939 fi
940 key=${1}
941 outputFile=${2}
942 [[ ${outputFile} =~ "-" ]] && outputFile=""
943 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
944 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
945 return 0
946}
947
948goCreateQAplots()
949{
950 umask 0002
951 mergedQAfileList=${1}
952 productionID=${2}
953 outputDir=${3}
954 configFile=${4}
955 shift 4
956 parseConfig ${configFile} ${@}
957
958 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
959
960 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
961 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
962 echo "${0} $*"
963
964 olddir=${PWD}
965 mkdir -p ${outputDir}
966 cd ${outputDir}
967 [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
968
969 echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
970 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
971
972 cd ${olddir}
973 return 0
974}
975
976goTest()
977{
978 umask 0002
979 exec 2>&1
980 exec > >(tee test.log)
981 echo "$@"
982 echo something
983 return 0
984}
985
986alirootInfo()
987{
988 umask 0002
989 # save aliroot repository info
990 [[ -z "${ALICE_ROOT}" ]] && return 1
991
992 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
993 echo "\${ROOTSYS}=${ROOTSYS}"
994 echo "\${PATH}=${PATH}"
995 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
996
997 pushd ${PWD}
998 cd ${ALICE_ROOT}
999 echo
1000
1001 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1002 git status
1003 echo ""
1004 echo ""
1005 git diff ${currentBranch}
1006 popd
1007 return 0
1008}
1009
1010setYear()
1011{
1012 #set the year
1013 # ${1} - year to be set
1014 # ${2} - where to set the year
1015 year1=$(guessYear ${1})
1016 year2=$(guessYear ${2})
1017 local path=${2}
1018 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1019 echo ${path}
1020 return 0
1021}
1022
1023guessPeriod()
1024{
1025 #guess the period from the path, pick the rightmost one
1026 local IFS="/"
1027 declare -a path=( ${1} )
1028 local dirDepth=${#path[*]}
1029 for ((x=${dirDepth}-1;x>=0;x--)); do
1030 local field=${path[${x}]}
1031 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1032 done
1033 echo ${period}
1034 return 0
1035}
1036
1037guessYear()
1038{
1039 #guess the year from the path, pick the rightmost one
1040 local IFS="/"
1041 declare -a path=( ${1} )
1042 local dirDepth=${#path[*]}
1043 for ((x=${dirDepth}-1;x>=0;x--)); do
1044 local field=${path[${x}]}
1045 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1046 done
1047 echo ${year}
1048 return 0
1049}
1050
1051guessRunNumber()
1052{
1053 #guess the run number from the path, pick the rightmost one
1054 #works for /path/foo/000123456/bar/...
1055 #and /path/foo.run123456.bar
1056 local IFS="/."
1057 declare -a path=( ${1} )
1058 local dirDepth=${#path[*]}
1059 for ((x=${dirDepth}-1;x>=0;x--)); do
1060 local field=${path[${x}]}
1061 field=${field/run/000}
1062 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1063 done
1064 echo ${runNumber}
1065 return 0
1066}
1067
1068validateLog()
1069{
1070 log=${1}
1071 errorConditions=(
1072 'There was a crash'
1073 'floating'
1074 'error while loading shared libraries'
1075 'std::bad_alloc'
1076 's_err_syswatch_'
1077 'Thread [0-9]* (Thread'
1078 'AliFatal'
1079 'core dumped'
1080 '\.C.*error:.*\.h: No such file'
1081 )
1082
1083 warningConditions=(
1084 'This is serious !'
1085 'rocVoltage out of range:'
1086 )
1087
1088 local logstatus=0
1089 local errorSummary=""
1090 local warningSummary=""
1091
1092 for ((i=0; i<${#errorConditions[@]};i++)); do
1093 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1094 [[ -n ${tmp} ]] && tmp+=" : "
1095 errorSummary+=${tmp}
1096 done
1097
1098 for ((i=0; i<${#warningConditions[@]};i++)); do
1099 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1100 [[ -n ${tmp} ]] && tmp+=" : "
1101 warningSummary+=${tmp}
1102 done
1103
1104 if [[ -n ${errorSummary} ]]; then
1105 echo "${errorSummary}"
1106 return 1
1107 fi
1108
1109 if [[ -n ${warningSummary} ]]; then
1110 echo "${warningSummary}"
1111 return 2
1112 fi
1113
1114 return 0
1115}
1116
1117summarizeLogs()
1118{
1119 #print a summary of logs
1120 logFiles=(
1121 "*.log"
1122 "stdout"
1123 "stderr"
1124 )
1125
1126 #check logs
1127 local logstatus=0
1128 for log in ${logFiles[*]}; do
1129 finallog=${PWD%/}/${log}
1130 [[ ! -f ${log} ]] && continue
1131 errorSummary=$(validateLog ${log})
1132 validationStatus=$?
1133 if [[ ${validationStatus} -eq 0 ]]; then
1134 #in pretend mode randomly report an error in rec.log some cases
1135 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1136 [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1137 else
1138 echo "${finallog} OK"
1139 fi
1140 elif [[ ${validationStatus} -eq 1 ]]; then
1141 echo "${finallog} BAD ${errorSummary}"
1142 elif [[ ${validationStatus} -eq 2 ]]; then
1143 echo "${finallog} OK MWAH ${errorSummary}"
1144 fi
1145 done
1146
1147 #report core files
1148 while read x; do
1149 echo ${x}
1150 chmod 644 ${x}
1151 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1152 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1153
1154 return ${logstatus}
1155}
1156
1157spitOutLocalOCDBaccessConfig()
1158{
1159 umask 0002
1160 #find ${1} -name "*root" | \
1161 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1162 while read line
1163 do
1164 local tmp=${line#${1}}
1165 echo ${tmp%/*} | \
1166 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1167 done
1168 return 0
1169}
1170
1171goMakeLocalOCDBaccessConfig()
1172{
1173 umask 0002
1174 # make a script that sets the specific storages form all the root files produced by CPass0
1175 local localOCDBpathCPass0=${1}
1176 local OCDBpathPrefix=${2}
1177 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1178
1179 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1180 tar xzf ${localOCDBpathCPass0}
1181 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1182 fi
1183
1184 echo
1185 echo creating the specific storage script
1186 echo localOCDBaccessConfig.C
1187 echo based on OCDB: ${localOCDBaccessConfig}
1188 echo
1189
1190 local tempLocalOCDB=""
1191 if [[ -f localOCDBaccessConfig.C ]]; then
1192 tempLocalOCDB=$(mktemp)
1193 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1194 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1195 fi
1196
1197 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1198 echo "{" >> localOCDBaccessConfig.C
1199 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
1200 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1201 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
1202 echo "}" >> localOCDBaccessConfig.C
1203
1204 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1205
1206 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1207 echo
1208 echo "!!!!!!! CPass0 produced no OCDB entries"
1209 return 1
1210 fi
1211 return 0
1212}
1213
1214goMakeFilteredTrees()
1215{
1216 outputDir=${1}
1217 runNumber=${2}
1218 #get path to input list
1219 inputListfiles=${3}
1220 #get scale number for tracks
1221 filterT=${4}
1222 #get scale number for V0s
1223 filterV=${5}
1224 #get OCDB path (optional)
1225 OCDBpath=${6}
1226 #get max number of files
1227 maxFiles=${7-"1000000"}
1228 #get offset of first file
1229 offsetFile=${8-"0"}
1230 #get max number of events
1231 maxEvents=${9-"30000000"}
1232 #get offset of first event
1233 offsetEvent=${10-"0"}
1234 configFile=${11-"benchmark.config"}
1235 esdFileName=${12-"AliESDs_Barrel.root"}
1236 shift 12
1237 parseConfig ${configFile} $@
1238
1239 commonOutputPath=${PWD}
1240 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1241
1242 runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
1243
1244 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1245
1246 mkdir -p ${outputDir}
1247 mkdir -p ${runpath}
1248
1249 cd ${runpath}
1250
1251 cat > filtering.log << EOF
1252 goMakeFilteredTrees config:
1253 runpath=${runpath}
1254 outputDir=${outputDir}
1255 commonOutputPath=${commonOutputPath}
1256 ALICE_ROOT=${ALICE_ROOT}
1257 PATH=${PATH}
1258 offsetEvent=$offsetEvent
1259 configFile=$configFile
1260 esdFileName=$esdFileName
1261EOF
1262
1263 if [[ -z ${pretend} ]];then
1264 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1265 else
1266 touch filtering.log FilterEvents_Trees.root
1267 fi
1268 pwd
1269 /bin/ls
1270 touch ${doneFile}
1271 summarizeLogs >> ${doneFile}
1272 echo mv -f * ${outputDir}
1273 mv -f * ${outputDir}
1274 [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1275 cd ${commonOutputPath}
1276 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1277 return 0
1278}
1279
1280submit()
1281{
1282 umask 0002
1283 [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1
1284 JobID=${1}
1285 startID=${2}
1286 endID=${3}
1287 waitForJOBID=${4}
1288 command=${5}
1289 commandArgs=${6}
1290
1291 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1292
1293 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1294
1295 batchSystem="SGE"
1296
1297 if [[ -z "${newFarm}" ]]
1298 then
1299 #old LSF
1300 # submit it (as job array)
1301 nFiles=$(( ${endID}-${startID}+1 ))
1302 while [ ${startID} -le ${nFiles} ] ; do
1303 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1304 endID=$(expr ${startID} + 999)
1305 else
1306 endID=${nFiles}
1307 fi
1308 if [[ ${waitForJOBID} -eq 0 ]]; then
1309 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1310 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1311 else
1312 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1313 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1314 fi
1315 startID=$(expr ${endID} + 1)
1316 done
1317 else
1318 #new SGE farm
1319 if [[ ${waitForJOBID} =~ "000" ]]; then
1320 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1321 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1322 else
1323 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1324 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1325 fi
1326 fi
1327 return 0
1328}
1329
1330goSubmitBatch()
1331{
1332 if [[ $# -lt 3 ]]; then
1333 echo "minimal use:"
1334 echo " ${0} submit fileList productionID configFile"
1335 return 0
1336 fi
1337
1338 productionID=${1}
1339 inputList=${2}
1340 configFile=${3:-"benchmark.config"}
1341 configFile=$(readlink -f ${configFile})
1342 shift 3
1343 extraOpts="$@"
1344 parseConfig ${configFile} ${extraOpts}
1345
1346 #redirect all output to submit.log
1347 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1348 exec 7>&1
1349 exec 1>submit_${productionID//"/"/_}.log 2>&1
1350
1351 umask 0002
1352 echo ${0}" submit $*"
1353 if [[ -z "${inputList}" || -z "${productionID}" ]]
1354 then
1355 echo
1356 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1357 echo
1358 return
1359 fi
1360
1361 # check if config file is there
1362 if [ ! -f ${configFile} ]; then
1363 echo "ERROR! Config File '${configFile}' not found" >&2
1364 return
1365 else
1366 echo "Using Config File: '${configFile}'"
1367 fi
1368
1369 # and print the configuration
1370 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1371
1372 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1373 self=$(readlink -f "${0}")
1374 configPath=$(dirname ${configFile})
1375 export commonOutputPath=${baseOutputDirectory}/${productionID}
1376
1377 mkdir -p ${commonOutputPath}
1378 mkdir -p ${commonOutputPath}/logs
1379 mkdir -p ${commonOutputPath}/meta
1380
1381 cp ${self} ${commonOutputPath}
1382 cp ${configFile} ${commonOutputPath}
1383 cp ${inputList} ${commonOutputPath}
1384 self=${commonOutputPath}/${self##*/}
1385 chmod u+x ${self}
1386 configFile=${commonOutputPath}/${configFile##*/}
1387 inputList=${commonOutputPath}/${inputList##*/}
1388
1389 #convert to absolut pathnames
1390 inputList=$(readlink -f "${inputList}")
1391 #make list of runs
1392 if [[ -z ${runNumber} ]]; then
1393 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1394 else
1395 listOfRuns=${runNumber}
1396 fi
1397
1398 alirootSource=$(readlink -f "${alirootSource}")
1399
1400 echo ""
1401 echo "### BEGIN CONFIGURATION ###"
1402 echo ""
1403 echo "GENERAL:"
1404 echo ""
1405 echo " productionID: ${productionID}"
1406 echo " batchCommand: ${batchCommand}"
1407 echo " batchFlags: ${batchFlags}"
1408 echo " alirootSource: ${alirootSource}"
1409 echo " ALICE_ROOT: ${ALICE_ROOT}"
1410 echo " ALIROOT_RELEASE: ${ALICE_RELEASE}"
1411 echo " inputList: ${inputList}"
1412 echo " configPath: ${configPath}"
1413 echo " commonOutputPath: ${commonOutputPath}"
1414 echo " defaultOCDB: ${defaultOCDB}"
1415 echo " autoOCDB: ${autoOCDB}"
1416 echo " recoTriggerOptions: ${recoTriggerOptions}"
1417 echo " runs:"
1418 echo " ${listOfRuns[*]}"
1419 echo ""
1420 echo "THE TRAIN WILL RUN:"
1421
1422 if [ ${runCPass0reco} -eq 1 ]; then
1423 echo " Pass0 - Recontruction"
1424 fi
1425
1426 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1427 echo " Pass0 - merging and OCDB export"
1428 fi
1429
1430 if [ ${runCPass1reco} -eq 1 ]; then
1431 echo " Pass1 - Recontruction"
1432 fi
1433 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1434 echo " Pass1 - merging and OCDB export"
1435 fi
1436
1437 echo ""
1438 echo "LIMITS:"
1439 echo " max. Events/Chunk: ${nEvents}"
1440 echo " max. Number of Chunks per Run: ${nMaxChunks}"
1441 echo ""
1442 echo "### END CONFIGURATION ###"
1443 echo ""
1444
1445
1446 # check if input file is there
1447 if [ ! -f ${inputList} ]; then
1448 echo "ERROR! Input List '${inputList}' not found" >&2
1449 return
1450 fi
1451
1452 # define jobid (for dependent jobs)
1453 date=$(date +%Y_%m_%d_%H%M%S)
1454 #for each run we submit one jobarray:
1455 for runNumber in ${listOfRuns[*]}; do
1456
1457 [[ -z ${runNumber} ]] && continue
1458 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1459
1460 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1461 JOBID1="p0_${JOBpostfix}"
1462 JOBID1wait="w0_${JOBpostfix}"
1463 JOBID2="m0_${JOBpostfix}"
1464 JOBID2wait="wm0_${JOBpostfix}"
1465 JOBID3="op0_${JOBpostfix}"
1466 JOBID3wait="wop0_${JOBpostfix}"
1467 JOBID4="p1_${JOBpostfix}"
1468 JOBID4wait="w1_${JOBpostfix}"
1469 JOBID5="m1_${JOBpostfix}"
1470 JOBID5wait="wm1_${JOBpostfix}"
1471 JOBID6="s1_${JOBpostfix}"
1472 JOBID6wait="ws1_${JOBpostfix}"
1473 JOBID7="QA_${JOBpostfix}"
1474 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1475 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1476 LASTJOB="000"
1477
1478 oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1479
1480 currentDefaultOCDB=${defaultOCDB}
1481 [[ -z ${autoOCDB} ]] && autoOCDB=1
1482 if [[ ${autoOCDB} -ne 0 ]]; then
1483 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1484 fi
1485
1486 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1487
1488 ###############################################################################
1489 #run one chunk with valgrind:
1490 if [[ -n ${runValgrind} ]]; then
1491 [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
1492 [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1493 submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}"
1494 fi
1495
1496 ################################################################################
1497 ################################################################################
1498 # run the CPass0 if requested
1499
1500 if [ ${runCPass0reco} -eq 1 ]; then
1501
1502 echo
1503 echo "starting CPass0... for run ${runNumber}"
1504 echo
1505
1506 # create directory and copy all files that are needed
1507 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1508 mkdir -p ${targetDirectory}
1509
1510 filesCPass0=(
1511 "${configPath}/runCPass0.sh"
1512 "${configPath}/recCPass0.C"
1513 "${configPath}/runCalibTrain.C"
1514 "${configPath}/localOCDBaccessConfig.C"
1515 "${configPath}/OCDB*.root"
1516 "${configPath}/sim.C"
1517 "${configPath}/rec.C"
1518 "${configPath}/Config.C"
1519 )
1520 for file in ${filesCPass0[*]}; do
1521 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1522 done
1523
1524 localInputList=${targetDirectory}/${inputList##*/}
1525 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1526 # limit nFiles to nMaxChunks
1527 nFiles=$(wc -l < ${localInputList})
1528 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1529 echo "raw files in list: ${nFiles}"
1530 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1531 nFiles=${nMaxChunks}
1532 fi
1533 echo "raw files to process: ${nFiles}"
1534 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1535 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1536 nFilesToWaitFor=${nFiles}
1537 else
1538 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1539 fi
1540 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1541 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1542
1543 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1544
1545 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1546 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1547 LASTJOB=${JOBID1wait}
1548
1549 fi #end running CPass0
1550 ################################################################################
1551
1552
1553 ################################################################################
1554 # submit merging of CPass0, depends on the reconstruction
1555
1556 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1557
1558 echo
1559 echo "submit CPass0 merging for run ${runNumber}"
1560 echo
1561
1562 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1563 mkdir -p ${targetDirectory}
1564
1565 #copy the scripts
1566 filesMergeCPass0=(
1567 "${configPath}/OCDB.root"
1568 "${configPath}/mergeMakeOCDB.byComponent.sh"
1569 "${configPath}/mergeMakeOCDB.sh"
1570 "${configPath}/localOCDBaccessConfig.C"
1571 "${configPath}/mergeByComponent.C"
1572 "${configPath}/makeOCDB.C"
1573 "${configPath}/merge.C"
1574 )
1575 for file in ${filesMergeCPass0[*]}; do
1576 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1577 done
1578
1579 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}"
1580 LASTJOB=${JOBID2}
1581
1582 if [[ -n ${generateMC} ]]; then
1583 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done"
1584 LASTJOB="mrl${JOBpostfix}"
1585 fi
1586
1587 echo
1588 fi
1589 # end of merging CPass0
1590 ################################################################################
1591
1592 ################################################################################
1593 ################################################################################
1594 # run the CPass1 if requested
1595
1596 if [ ${runCPass1reco} -eq 1 ]; then
1597
1598 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1599 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1600
1601 # safety feature: if we are re-running for any reason we want to delete the previous output first.
1602 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1603
1604 echo
1605 echo "starting CPass1... for run ${runNumber}"
1606 echo
1607
1608 # create directory and copy all files that are needed
1609 mkdir -p ${targetDirectory}
1610
1611 filesCPass1=(
1612 "${configPath}/runCPass1.sh"
1613 "${configPath}/recCPass1.C"
1614 "${configPath}/recCPass1_OuterDet.C"
1615 "${configPath}/runCalibTrain.C"
1616 "${configPath}/QAtrain_duo.C"
1617 "${configPath}/localOCDBaccessConfig.C"
1618 "${configPath}/OCDB.root"
1619 )
1620 for file in ${filesCPass1[*]}; do
1621 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1622 done
1623
1624 if [[ -n ${generateMC} ]]; then
1625 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1626 else
1627 localInputList=${targetDirectory}/${inputList##*/}
1628 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1629 fi
1630 # limit nFiles to nMaxChunks
1631 nFiles=$(wc -l < ${localInputList})
1632 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1633 echo "raw files in list: ${nFiles}"
1634 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1635 nFiles=${nMaxChunks}
1636 fi
1637 echo "raw files to process: ${nFiles}"
1638 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1639 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1640 nFilesToWaitFor=${nFiles}
1641 else
1642 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1643 fi
1644 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1645 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1646
1647 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1648
1649 ################################################################################
1650 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1651 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1652 LASTJOB=${JOBID4wait}
1653 ################################################################################
1654
1655 echo
1656 fi #end running CPass1
1657
1658 ################################################################################
1659 # submit merging of CPass1, depends on the reconstruction
1660 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1661
1662 echo
1663 echo "submit CPass1 merging for run ${runNumber}"
1664 echo
1665
1666 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1667 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1668 mkdir -p ${targetDirectory}
1669
1670 # copy files
1671 filesMergeCPass1=(
1672 "${configPath}/OCDB.root"
1673 "${configPath}/localOCDBaccessConfig.C"
1674 "${configPath}/mergeMakeOCDB.byComponent.sh"
1675 "${configPath}/mergeByComponent.C"
1676 "${configPath}/makeOCDB.C"
1677 "${configPath}/merge.C"
1678 "${configPath}/mergeMakeOCDB.sh"
1679 "${configPath}/QAtrain_duo.C"
1680 )
1681 for file in ${filesMergeCPass1[*]}; do
1682 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1683 done
1684
1685 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}"
1686 LASTJOB=${JOBID5}
1687 echo
1688 fi
1689
1690 ##############################
1691 if [ ${runESDfiltering} -eq 1 ]; then
1692 rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1693 rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1694 echo
1695 echo submitting filtering for run ${runNumber}
1696 echo
1697 submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done "
1698 submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
1699 LASTJOB=${JOBfilterESDcpass1}
1700 fi
1701
1702 done
1703
1704 #################################################################################
1705 #################################################################################
1706 if [ ${runESDfiltering} -eq 1 ]; then
1707 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1708 else
1709 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1710 fi
1711 LASTJOB=${JOBID5wait}
1712
1713 #################################################################################
1714 echo
1715 echo "submit make a summary"
1716 echo
1717
1718 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}"
1719 LASTJOB=${JOBID6}
1720 #################################################################################
1721
1722 #restore stdout
1723 exec 1>&7 7>&-
1724 echo "jobs submitted."
1725 return 0
1726}
1727
1728goWaitForOutput()
1729{
1730 umask 0002
1731 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1732 echo searchPath=${1}
1733 echo fileName=${2}
1734 echo numberOfFiles=${3}
1735 echo maxSecondsToWait=${4}
1736 searchPath=${1}
1737 fileName=${2}
1738 numberOfFiles=${3}
1739 maxSecondsToWait=${4}
1740 extraFindOptions=${5}
1741 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1742 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1743 while true; do
1744 #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1745 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1746 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1747 [[ ${n} -ge ${numberOfFiles} ]] && break
1748 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1749 sleep 60
1750 done
1751 echo "DONE! exiting..."
1752 return 0
1753}
1754
1755mergeSysLogs()
1756{
1757 outputFile=${1}
1758 shift
1759 inputFiles="$@"
1760 i=0
1761 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1762 while read x; do
1763 runNumber=$(guessRunNumber ${x})
1764 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1765 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1766 (( i++ ))
1767 done < <(ls -1 ${inputFiles}) > ${outputFile}
1768 return 0
1769}
1770
1771goMakeMergedSummaryTree()
1772{
1773 # create list of calibration entries
1774 # takes no arguments, just run it in the base output
1775 # directory with the following files in the working directory
1776 #
1777 # Calibration file lists:
1778 # cpass0.dcsTree.list, cpass1.dcsTree.list
1779 # QA trending root file:
1780 # trending.root
1781 #
1782 # Production infoo ascii files:
1783 # summary_pass0.tree
1784 # summary_pass1.tree
1785 #
1786
1787 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1788 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1789 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1790 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1791 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1792
1793 #first, dump the C macro to file
1794 cat << EOF > mergeTree.C
1795 //
1796 // Merge summary information
1797 // Following files are expected to be in the working directory
1798 //
1799 // Calibration file lists:
1800 // cpass0.dcsTree.list, cpass1.dcsTree.list
1801 // QA trending root file:
1802 // trending.root
1803 //
1804 // Production infoo ascii files:
1805 // summary_pass0.tree
1806 // summary_pass1.tree
1807 //
1808 void mergeTree(){
1809 //
1810 //
1811 //
1812 // Calibration values dump
1813 //
1814 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1815 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1816 //
1817 // Calibration status dump
1818 //
1819 TFile *fprod = TFile::Open("fproduction.root","recreate");
1820 TTree tree0, tree1;
1821 tree0.ReadFile("summary_pass0.tree");
1822 tree1.ReadFile("summary_pass1.tree");
1823 tree0.Write("CPass0");
1824 tree1.Write("CPass1");
1825 fprod->Close();
1826 //
1827 //
1828 //
1829 TString stringSetup="";
1830 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
1831 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
1832 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
1833 //
1834 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
1835 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
1836 //
1837 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1838 }
1839EOF
1840
1841 aliroot -b -q "mergeTree.C" > mergeTrees.log
1842 return $?
1843}
1844
1845goMakeSummary()
1846{
1847 #all the final stuff goes in here for ease of use:
1848 # summary logs
1849 # qa plot making
1850 # final file lists
1851
1852 configFile=${1}
1853 shift 1
1854 extraOpts="$@"
1855 log="summary.log"
1856 productionID="qa"
1857 parseConfig ${configFile} ${extraOpts}
1858
1859 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1860
1861 exec &> >(tee ${log})
1862
1863 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1864
1865 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1866
1867 #summarize the global stuff
1868 echo "env script: ${alirootSource} ${alirootEnv}"
1869 echo "\$ALICE_ROOT=${ALICE_ROOT}"
1870 echo "commonOutputPath=${commonOutputPath}"
1871
1872 #summarize the stacktraces
1873 awk '
1874 BEGIN {
1875 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1876 RS="#[0-9]*";
1877 aliroot=0;
1878 }
1879 {
1880 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1881 gsub("#","",RT);
1882 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1883 }
1884 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1885 awk '
1886 BEGIN {
1887 RS="#[0-9]*";
1888 aliroot=0;
1889 }
1890 {
1891 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1892 gsub("#","",RT);
1893 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1894 }
1895 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1896
1897 echo total numbers for the production:
1898 echo
1899 awk 'BEGIN {nFiles=0;nCore=0;}
1900 /^calibfile/ {nFiles++;}
1901 /core dumped/ {nCore++i;}
1902 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1903 awk 'BEGIN {nOK=0; nBAD=0; }
1904 /\/rec.log OK/ {nOK++;}
1905 /\/rec.log BAD/ {nBAD++;}
1906 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1907 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1908 awk 'BEGIN {nOK=0; nBAD=0; }
1909 /\/calib.log OK/ {nOK++;}
1910 /\/calib.log BAD/ {nBAD++;}
1911 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1912
1913 awk 'BEGIN {nOK=0; nBAD=0; }
1914 /merge.log OK/ {nOK++;}
1915 /merge.log BAD/ {nBAD++;}
1916 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1917 awk 'BEGIN {nOK=0; nBAD=0; }
1918 /ocdb.log OK/ {nOK++;}
1919 /ocdb.log BAD/ {nBAD++;}
1920 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1921
1922 echo
1923 awk 'BEGIN {nFiles=0;nCore=0;}
1924 /^calibfile/ {nFiles++;}
1925 /core dumped/ {nCore++;}
1926 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1927 awk 'BEGIN {nOK=0; nBAD=0; }
1928 /\/rec.log OK/ {nOK++;}
1929 /\/rec.log BAD/ {nBAD++;}
1930 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1931 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1932 awk 'BEGIN {nOK=0; nBAD=0; }
1933 /\/calib.log OK/ {nOK++;}
1934 /\/calib.log BAD/ {nBAD++;}
1935 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1936
1937 awk 'BEGIN {nOK=0; nBAD=0; }
1938 /merge.log OK/ {nOK++;}
1939 /merge.log BAD/ {nBAD++;}
1940 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1941 awk 'BEGIN {nOK=0; nBAD=0; }
1942 /ocdb.log OK/ {nOK++;}
1943 /ocdb.log BAD/ {nBAD++;}
1944 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1945
1946 echo
1947 echo per run stats:
1948 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
1949do
1950 dir=$(goPrintValues calibfile - ${x})
1951 runNumber=$(guessRunNumber ${dir})
1952 [[ -z ${runNumber} ]] && continue
1953
1954 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
1955 statusCPass0=( $(
1956 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
1957 /\/rec.log OK/ {nOKrec++;}
1958 /\/rec.log BAD/ {nBADrec++;}
1959 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
1960 /stderr OK/ {nOKstderr++;}
1961 /\/calib.log OK/ {nOKcalib++;}
1962 /\/calib.log BAD/ {nBADcalib++}
1963 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
1964 ) )
1965 fi
1966
1967 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
1968 statusCPass1=( $(
1969 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
1970 /\/rec.log OK/ {nOKrec++;}
1971 /\/rec.log BAD/ {nBADrec++;}
1972 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
1973 /stderr OK/ {nOKstderr++;}
1974 /\/calib.log OK/ {nOKcalib++;}
1975 /\/calib.log BAD/ {nBADcalib++}
1976 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
1977 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
1978 /\/qa_outer.log OK/ {nQAouterOK++;}
1979 /\/qa_outer.log BAD/ {nQAouterBAD++;}
1980 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
1981 ) )
1982 fi
1983
1984 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
1985 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1986 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1987
1988 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
1989done
1990
1991 #make lists with output files - QA, trending, filtering and calibration
1992 ### wait for the merging of all runs to be over ###
1993 rm -f qa.list
1994 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1995 rm -f calib.list
1996 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1997 rm -f trending.list
1998 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1999 rm -f filtering.list
2000 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null
2001 #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2002 rm -f cpass0.dcsTree.list
2003 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2004 #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2005 rm -f cpass1.dcsTree.list
2006 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2007
2008 #merge trending
2009 rm -f ${commonOutputPath}/trending_merged.root
2010 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log
2011
2012 goMakeSummaryTree ${commonOutputPath} 0
2013 goMakeSummaryTree ${commonOutputPath} 1
2014
2015 goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log
2016
2017 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2018 goMakeMergedSummaryTree
2019
2020 #if set, email the summary
2021 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2022
2023 return 0
2024}
2025
2026goMakeSummaryTree()
2027{
2028 if [[ $# -lt 1 ]] ; then
2029 return
2030 fi
2031 #1. define vars/arrays
2032 DIR=${1} #use input or exec in current dir
2033 pass=${2-"0"} #pass from input
2034 outfile="summary_pass${pass}.tree"
2035 Ncolumns=0
2036 test -f ${outfile} && : >${outfile}
2037 errfile=${outfile/tree/err}
2038 test -f ${errfile} && : >${errfile}
2039
2040 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2041 Ncounter=${#counterString[@]}
2042
2043 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2044 Nstatus=${#statusString[@]}
2045
2046
2047 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2048 Nrates=${#ratesString[@]}
2049
2050 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2051 Nruns=${#runs[@]}
2052
2053 echo -n runnumber/I >>${outfile}
2054 echo -n :cpass${pass}status/I >>${outfile}
2055 echo -n :cpass${pass}QAstatus/I >>${outfile}
2056 for i in ${ratesString[@]}; do
2057 echo -n :${i}OK/I >>${outfile}
2058 echo -n :${i}BAD/I >>${outfile}
2059
2060 done
2061 for i in ${counterString[@]} ${statusString[@]} ; do
2062 echo -n :${i}/I >>${outfile}
2063 done
2064 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2065 echo >> ${outfile}
2066
2067 #2. loop runs
2068
2069 for runnumber in ${runs[@]} ; do
2070
2071
2072
2073 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2074 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2075 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2076 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2077 echo "${filemerge} does not exist!" >>${errfile}
2078 continue
2079 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2080 echo "${filejob} does not exist!" >>${errfile}
2081 echo -n ${runnumber} >> ${outfile}
2082 for i in $(seq ${Ncolumns}) ; do
2083 echo -n "-1" >> ${outfile}
2084 done
2085 echo >> ${outfile}
2086 continue
2087 fi
2088 echo -n ${runnumber} >> ${outfile}
2089 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2090 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2091 echo -n " ${passStatus}" >> ${outfile}
2092 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2093 echo -n " ${qaStatus}" >> ${outfile}
2094
2095
2096 #fill OK/BAD rates
2097 for i in $(seq 0 $((${Nrates}-1))) ; do
2098 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2099 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2100
2101 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2102 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2103 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2104 fi
2105 echo -n " ${var1}" >> ${outfile}
2106 echo -n " ${var2}" >> ${outfile}
2107 done
2108
2109 if [[ -f ${fileOCDB} ]] ; then
2110 #fill counter
2111 for i in $(seq 0 $((${Ncounter}-1))) ; do
2112 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2113 echo -n " ${var1:-"-1"}" >> ${outfile}
2114 done
2115
2116 #fill status
2117 for i in $(seq 0 $((${Nstatus}-1))) ; do
2118 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2119 echo -n " ${var1:-"-1"}" >> ${outfile}
2120 done
2121 fi
2122 echo >> ${outfile}
2123 done
2124
2125 return 0
2126}
2127
2128parseConfig()
2129{
2130 configFile=${1}
2131 shift
2132
2133 #first, source the config file
2134 if [ -f ${configFile} ]; then
2135 source ${configFile}
2136 else
2137 echo "config file ${configFile} not found!, skipping..."
2138 fi
2139
2140 #then, parse the options as theya override the optionf from file
2141 while [[ -n ${1} ]]; do
2142 var="${1#--}"
2143 echo "extraOpts: exporting ${var}"
2144 eval "export ${var}"
2145 shift
2146 done
2147
2148 #export the aliroot function if defined to override normal behaviour
2149 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2150 return 0
2151}
2152
2153aliroot()
2154{
2155 args="$@"
2156 if [[ -n ${valgrindCommand} ]]; then
2157 valgrindLogFile="cpu.txt"
2158 [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2159 [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2160 [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
2161 echo running ${valgrindCommand} aliroot ${args}
2162 ${valgrindCommand} aliroot ${args}
2163 else
2164 #to prevent an infinite recursion use "command aliroot" to disable
2165 #aliases and functions
2166 echo running command aliroot ${args}
2167 command aliroot ${args}
2168 fi
2169 return 0
2170}
2171
2172guessRunData()
2173{
2174 #guess the period from the path, pick the rightmost one
2175 period=""
2176 runNumber=""
2177 year=""
2178 pass=""
2179 legoTrainRunNumber=""
2180 dataType=""
2181
2182 local shortRunNumber=""
2183 local IFS="/"
2184 declare -a path=( $1 )
2185 local dirDepth=$(( ${#path[*]}-1 ))
2186 i=0
2187 #for ((x=${dirDepth};x>=0;x--)); do
2188 for ((x=0;x<=${dirDepth};x++)); do
2189
2190 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2191 local field=${path[${x}]}
2192 local fieldNext=${path[$((x+1))]}
2193
2194 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2195 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2196 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2197 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2198 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2199 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2200 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2201 (( i++ ))
2202 done
2203 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2204 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2205
2206 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2207 if [[ -z ${runNumber}} ]];
2208 then
2209 #error condition
2210 return 1
2211 else
2212 #ALL OK
2213 return 0
2214 fi
2215 return 0
2216}
2217
2218main "$@"