]> git.uio.no Git - u/mrichter/AliRoot.git/blame_incremental - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
react to filename change *_Barrel -> *_barrel
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
... / ...
CommitLineData
1#!/bin/bash
2#include benchmark.config
3
4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
24 umask 0002
25 shift
26 case ${runMode} in
27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33 "MakeSummary") goMakeSummary "$@";;
34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
36 "test") goTest "$@";;
37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
41 "Merge") goMerge "$@";;
42 *)
43 ${runMode} "$@"
44 ;;
45 esac
46 return 0
47}
48
49generateMC()
50{
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
54 runNumber=${1}
55 OCDBpath=${2}
56 nEventsim=${3}
57 if [[ -n ${pretend} ]]; then
58 touch galice.root
59 else
60 if [[ -f sim.C && -f Config.C ]] ; then
61 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
62 mv syswatch.log simwatch.log
63 fi
64 fi
65}
66
67goCPass0()
68{
69 umask 0002
70
71 targetDirectory=${1}
72 inputList=${2}
73 nEvents=${3}
74 ocdbPath=${4}
75 configFile=${5}
76 runNumber=${6}
77 jobindex=${7}
78 shift 7
79 if ! parseConfig ${configFile} "$@"; then return 1; fi
80
81 #record the working directory provided by the batch system
82 batchWorkingDirectory=${PWD}
83
84 #use the jobindex only if set and non-negative
85 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
86 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
87 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
88 fi
89
90 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
91 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
92
93 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
94
95 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
96 ulimit -c unlimited
97 export ALIROOT_FORCE_COREDUMP
98 fi
99
100 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
101 #at the QA stage the pass number is guessed from the path stored here.
102 #The Format is:
103 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
104 export PRODUCTION_METADATA="OutputDir=cpass0"
105
106 if [[ "${inputList}" =~ \.root$ ]]; then
107 infile=${inputList}
108 else
109 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
110 fi
111 chunkName=${infile##*/}
112
113 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
114 mkdir -p ${outputDir}
115 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
116
117 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
118 runpath=${outputDir}
119 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
120 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
121 mkdir -p ${runpath}
122 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
123 cd ${runpath}
124 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
125
126 #runCPassX/C expects the raw chunk to be linked in the run dir
127 #despite it being accessed by the full path
128 ln -s ${infile} ${runpath}/${chunkName}
129
130 #####MC
131 if [[ -n ${generateMC} ]]; then
132 olddir=${PWD}
133 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
134 simrunpath=${outputDirMC}
135 [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
136 [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
137 mkdir -p ${outputDirMC}
138 mkdir -p ${simrunpath}
139 cd ${simrunpath}
140
141 filesMC=(
142 "${batchWorkingDirectory}/sim.C"
143 "${batchWorkingDirectory}/rec.C"
144 "${batchWorkingDirectory}/Config.C"
145 "${batchWorkingDirectory}/OCDB_*.root"
146 )
147 for file in ${filesMC[*]}; do
148 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
149 done
150
151 generateMC ${runNumber} ${ocdbPath} ${nEvents}
152
153 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
154 cd ${olddir}
155
156 ln -s ${outputDirMC}/* ${runpath}/
157
158 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
159 infile=""
160 fi
161 ######
162
163 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
164
165 logOutputDir=${runpath}
166 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
167 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
168 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
169 echo "${0} $*"
170
171 echo "#####################"
172 echo CPass0:
173 echo JOB setup
174 echo nEvents ${nEvents}
175 echo runNumber ${runNumber}
176 echo ocdbPath ${ocdbPath}
177 echo infile ${infile}
178 echo chunkName ${chunkName}
179 echo jobindex ${jobindex}
180 echo recoTriggerOptions ${recoTriggerOptions}
181 echo targetDirectory ${targetDirectory}
182 echo commonOutputPath ${commonOutputPath}
183 echo doneFile ${doneFile}
184 echo runpath ${runpath}
185 echo outputDir ${outputDir}
186 echo ALICE_ROOT ${ALICE_ROOT}
187 echo PWD ${PWD}
188 echo "########## ###########"
189
190 alirootInfo > ALICE_ROOT.log
191
192 filesCPass0=(
193 "${batchWorkingDirectory}/runCPass0.sh"
194 "${batchWorkingDirectory}/recCPass0.C"
195 "${batchWorkingDirectory}/runCalibTrain.C"
196 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
197 "${batchWorkingDirectory}/OCDB.root"
198 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
199 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
200 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
201 )
202
203 for file in ${filesCPass0[*]}; do
204 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
205 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
206 done
207
208 echo "this directory (${PWD}) contents:"
209 /bin/ls
210 echo
211 chmod u+x runCPass0.sh
212
213 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
214
215 if [[ -n ${postSetUpActionCPass0} ]]; then
216 echo "running ${postSetUpActionCPass0}"
217 eval ${postSetUpActionCPass0}
218 fi
219
220 #run CPass0
221 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
222 if [[ -n ${pretend} ]]; then
223 touch AliESDs.root
224 touch AliESDfriends.root
225 touch AliESDfriends_v1.root
226 touch rec.log
227 touch calib.log
228 else
229 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
230 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
231 fi
232
233 #move stuff to final destination
234 echo "this directory (${PWD}) contents:"
235 /bin/ls
236 echo
237
238 echo rm -f ./${chunkName}
239 rm -f ./${chunkName}
240 echo "cp --recursive ${runpath}/* ${outputDir}"
241 cp -p --recursive ${runpath}/* ${outputDir}
242 echo
243
244 #validate CPass0
245 cd ${outputDir}
246 touch ${doneFile}
247 echo "dir ${outputDir}" >> ${doneFile}
248 if summarizeLogs >> ${doneFile}; then
249 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
250 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
251 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
252 fi
253
254 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
255 return 0
256}
257
258goCPass1()
259{
260 umask 0002
261
262 targetDirectory=${1}
263 inputList=${2}
264 nEvents=${3}
265 ocdbPath=${4}
266 configFile=${5}
267 runNumber=${6}
268 jobindex=${7}
269 shift 7
270 extraOpts=("$@")
271 if ! parseConfig ${configFile} "$@"; then return 1; fi
272
273 #record the working directory provided by the batch system
274 batchWorkingDirectory=${PWD}
275
276 #use the jobindex only if set and non-negative
277 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
278 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
279 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
280 fi
281
282 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
283 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
284
285 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
286
287 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
288 ulimit -c unlimited
289 export ALIROOT_FORCE_COREDUMP
290 fi
291
292 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
293 #at the QA stage the pass number is guessed from the path stored here.
294 #The Format is:
295 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
296 export PRODUCTION_METADATA="OutputDir=cpass1"
297
298 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
299 if [[ "${inputList}" =~ \.root$ ]]; then
300 infile=${inputList}
301 else
302 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
303 fi
304 chunkName=${infile##*/}
305
306 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
307 mkdir -p ${outputDir}
308 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
309
310 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
311 runpath=${outputDir}
312 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
313 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
314
315 #MC
316 if [[ "${infile}" =~ galice\.root ]]; then
317 ln -s ${inputList%/*}/* ${runpath}
318 infile=""
319 fi
320
321 #init the running path
322 mkdir -p ${runpath}
323 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
324 cd ${runpath}
325 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
326
327 #this is needed for runCPass1.sh
328 ln -s ${infile} ${runpath}/${chunkName}
329
330 logOutputDir=${runpath}
331 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
332 [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
333 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
334 echo "${0} $*"
335
336 echo "#####################"
337 echo CPass1:
338 echo JOB setup
339 echo nEvents ${nEvents}
340 echo runNumber ${runNumber}
341 echo ocdbPath ${ocdbPath}
342 echo infile ${infile}
343 echo chunkName ${chunkName}
344 echo jobindex ${jobindex}
345 echo recoTriggerOptions ${recoTriggerOptions}
346 echo targetDirectory ${targetDirectory}
347 echo commonOutputPath ${commonOutputPath}
348 echo doneFile ${doneFile}
349 echo runpath ${runpath}
350 echo outputDir ${outputDir}
351 echo ALICE_ROOT ${ALICE_ROOT}
352 echo PWD ${PWD}
353 echo "########## ###########"
354
355 alirootInfo > ALICE_ROOT.log
356
357 filesCPass1=(
358 "${batchWorkingDirectory}/runCPass1.sh"
359 "${batchWorkingDirectory}/recCPass1.C"
360 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
361 "${batchWorkingDirectory}/runCalibTrain.C"
362 "${batchWorkingDirectory}/QAtrain_duo.C"
363 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
364 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
365 "${batchWorkingDirectory}/OCDB.root"
366 "${trustedQAtrainMacro}"
367 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
368 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
369 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
370 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
371 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
372 )
373
374 for file in ${filesCPass1[*]}; do
375 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
376 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
377 done
378
379 echo "this directory (${PWD}) contents:"
380 /bin/ls
381 echo
382
383 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
384
385 if [[ -n ${postSetUpActionCPass1} ]]; then
386 echo "running ${postSetUpActionCPass1}"
387 eval ${postSetUpActionCPass1}
388 echo
389 fi
390
391 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
392 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
393 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
394 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
395 else
396 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
397 fi
398
399 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
400 echo "cpass0 produced no calibration! exiting..."
401 touch ${doneFile}
402 return 1
403 fi
404
405 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
406 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
407 #entries using a relative path, e.g. local://./OCDB
408 mkdir Barrel OuterDet
409 ln -s ../OCDB Barrel/OCDB
410 ln -s ../OCDB OuterDet/OCDB
411
412 #setup the filtering
413 #the following option enables the filtering task inside the QAtrain_duo.C
414 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
415 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
416 if [[ -n ${filteringFactorHighPt} ]]; then
417 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
418 fi
419 if [[ -n ${filteringFactorV0s} ]]; then
420 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
421 fi
422
423 #run CPass1
424 chmod u+x runCPass1.sh
425 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
426 if [[ -n ${pretend} ]]; then
427 touch AliESDs_Barrel.root
428 touch AliESDfriends_Barrel.root
429 touch AliESDfriends_v1.root
430 touch QAresults_barrel.root
431 touch EventStat_temp_barrel.root
432 touch AODtpITS.root
433 touch AliESDs_Outer.root
434 touch AliESDfriends_Outer.root
435 touch QAresults_outer.root
436 touch EventStat_temp_outer.root
437 touch rec.log
438 touch calib.log
439 touch qa.log
440 touch filtering.log FilterEvents_Trees.root
441 else
442 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
443
444 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
445 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
446 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
447 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
448 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
449 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
450 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
451 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
452
453 #make the filtered tree (if requested and not already produced by QA
454 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
455 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
456 goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
457 fi
458
459 fi
460
461 ##handle possible crashes in QA (happens often in trunk)
462 ##rerun QA with a trusted aliroot version
463 #if [[ $(validateLog qa_barrel.log) ]]; then
464 # echo "qa_barrel.log not validated!"
465 #fi
466 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
467 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
468 # source ${setupTrustedAliROOTenvInCurrentShell}
469 # cd Barrel
470 # rm QAresults_barrel.root
471 # rm EventStat_temp_barrel.root
472 # rm AODtpITS.root
473 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
474 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
475 # if [[ -n ${trustedQAtrainMacro} ]]; then
476 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
477 # fi
478 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
479 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
480 # cd ../
481 #fi
482
483 #move stuff to final destination
484 echo "this directory (${PWD}) contents:"
485 /bin/ls
486 echo rm -f ./${chunkName}
487 rm -f ./${chunkName}
488 echo "cp --recursive ${runpath}/* ${outputDir}"
489 cp -pf --recursive ${runpath}/* ${outputDir}
490 echo
491
492 #validate CPass1
493 cd ${outputDir}
494 touch ${doneFile}
495 echo "dir ${outputDir}" >> ${doneFile}
496 if summarizeLogs >> ${doneFile}; then
497 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
498 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
499 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
500 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
501 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
502 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
503 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
504 fi
505
506 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
507 return 0
508}
509
510
511goMergeCPass0()
512{
513 #
514 # find the output files and merge them
515 #
516
517 outputDir=${1}
518 ocdbStorage=${2}
519 configFile=${3}
520 runNumber=${4}
521 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
522 shift 5
523 if ! parseConfig ${configFile} "$@"; then return 1; fi
524
525 #record the working directory provided by the batch system
526 batchWorkingDirectory=${PWD}
527
528 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
529 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
530
531 umask 0002
532 ulimit -c unlimited
533
534 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
535
536 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
537 runpath=${outputDir}
538 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
539 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
540
541 mkdir -p ${runpath}
542 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
543 cd ${runpath}
544 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
545
546 logOutputDir=${runpath}
547 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
548 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
549 echo "${0} $*"
550
551 mergingScript="mergeMakeOCDB.byComponent.sh"
552
553 echo goMergeCPass0 SETUP:
554 echo runNumber=${runNumber}
555 echo outputDir=${outputDir}
556 echo ocdbStorage=${ocdbStorage}
557 echo calibrationFilesToMerge=${calibrationFilesToMerge}
558 echo mergingScript=${mergingScript}
559 echo commonOutputPath=${commonOutputPath}
560 echo runpath=${runpath}
561
562 # copy files in case they are not already there
563 filesMergeCPass0=(
564 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
565 "${batchWorkingDirectory}/OCDB.root"
566 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
567 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
568 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
569 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
570 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
571 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
572 )
573 for file in ${filesMergeCPass0[*]}; do
574 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
575 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
576 done
577
578 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
579
580 alirootInfo > ALICE_ROOT.log
581
582 #
583 echo "PWD"
584 /bin/ls
585 echo "PWD/.."
586 /bin/ls ../
587
588
589 #merge calibration
590 chmod u+x ${mergingScript}
591 mkdir -p ./OCDB
592 if [[ ! -f ${calibrationFilesToMerge} ]]; then
593 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
594 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
595 fi
596
597 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
598 if [[ -n ${pretend} ]]; then
599 touch CalibObjects.root
600 touch ocdb.log
601 touch merge.log
602 touch dcsTime.root
603 mkdir -p ./OCDB/TPC/Calib/TimeGain/
604 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
605 touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
606 touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
607 else
608 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
609
610 #produce the calib trees for expert QA (dcsTime.root)
611 goMakeLocalOCDBaccessConfig ./OCDB
612 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
613 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
614 fi
615
616 ### produce the output
617 #tar the produced OCDB for reuse
618 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
619
620 /bin/ls
621
622 #copy all to output dir
623 cp -pf --recursive ${runpath}/* ${outputDir}
624
625 if [[ -n ${generateMC} ]]; then
626 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
627 fi
628
629 #validate merging cpass0
630 cd ${outputDir}
631 touch ${doneFile}
632 echo "dir ${outputDir}" >> ${doneFile}
633 if summarizeLogs >> ${doneFile}; then
634 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
635 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
636 fi
637
638 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
639 return 0
640}
641
642goMergeCPass1()
643{
644 #
645 # find the output files and merge them
646 #
647
648 outputDir=${1}
649 ocdbStorage=${2}
650 configFile=${3}
651 runNumber=${4}
652 calibrationFilesToMerge=${5}
653 qaFilesToMerge=${6}
654 filteredFilesToMerge=${7}
655 shift 7
656 if ! parseConfig ${configFile} "$@"; then return 1; fi
657
658 #record the working directory provided by the batch system
659 batchWorkingDirectory=${PWD}
660
661 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
662 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
663
664 umask 0002
665 ulimit -c unlimited
666
667 #clean up first:
668 rm -f ${outputDir}/*.log
669 rm -f ${outputDir}/*.root
670 rm -f ${outputDir}/*done
671
672 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
673
674 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
675 runpath=${outputDir}
676 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
677 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
678
679 mkdir -p ${runpath}
680 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
681 cd ${runpath}
682 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
683
684 logOutputDir=${runpath}
685 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
686 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
687 echo "${0} $*"
688
689 calibrationOutputFileName='AliESDfriends_v1.root'
690 qaOutputFileName='QAresults*.root'
691 mergingScript="mergeMakeOCDB.byComponent.sh"
692 #important to have the string "Stage.txt" in the filename to trigger the merging
693 #it has to be a list of directories containing the files
694 qaMergedOutputFileName="QAresults_merged.root"
695
696 echo goMergeCPass1 SETUP:
697 echo runNumber=${runNumber}
698 echo outputDir=${outputDir}
699 echo ocdbStorage=${ocdbStorage}
700 echo calibrationFilesToMerge=$calibrationFilesToMerge
701 echo qaFilesToMerge=$qaFilesToMerge
702 echo calibrationOutputFileName=${calibrationOutputFileName}
703 echo mergingScript=${mergingScript}
704
705 # copy files in case they are not already there
706 filesMergeCPass1=(
707 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
708 "${batchWorkingDirectory}/${qaFilesToMerge}"
709 "${batchWorkingDirectory}/OCDB.root"
710 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
711 "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
712 "${batchWorkingDirectory}/QAtrain_duo.C"
713 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
714 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
715 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
716 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
717 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
718 "${trustedQAtrainMacro}"
719 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
720 )
721 for file in ${filesMergeCPass1[*]}; do
722 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
723 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
724 done
725
726 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
727
728 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
729 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
730 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
731 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
732 else
733 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
734 fi
735
736 alirootInfo > ALICE_ROOT.log
737
738 #
739 /bin/ls
740
741 #merge calibration
742 chmod u+x ${mergingScript}
743 mkdir -p OCDB
744
745 #if not provided, create the lists of files to merge
746 if [[ ! -f ${filteredFilesToMerge} ]]; then
747 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
748 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
749 fi
750 if [[ ! -f ${calibrationFilesToMerge} ]]; then
751 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
752 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
753 fi
754 if [[ ! -f ${qaFilesToMerge} ]]; then
755 #find the files, but only store the directories (QAtrain_duo.C requires this)
756 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
757 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
758 fi
759
760 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
761 if [[ -n ${pretend} ]]; then
762 touch ocdb.log
763 touch cpass1.localOCDB.${runNumber}.tgz
764 touch ${qaMergedOutputFileName}
765 touch merge.log
766 touch trending.root
767 touch FilterEvents_Trees.root
768 touch CalibObjects.root
769 touch dcsTime.root
770 touch ${qaMergedOutputFileName}
771 mkdir -p OCDB
772 else
773 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
774
775 #merge QA (and filtered trees)
776 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
777 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
778
779 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
780 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
781 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
782 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
783 mv QAresults_barrel.root ${qaMergedOutputFileName}
784 mv trending_barrel.root trending.root
785
786 #merge filtered trees
787 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
788 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
789
790 #produce the calib trees for expert QA
791 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
792 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
793 fi
794
795 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
796
797 /bin/ls
798
799 #copy all to output dir
800 cp -pf --recursive ${runpath}/* ${outputDir}
801
802 #validate merge cpass1
803 cd ${outputDir}
804 touch ${doneFile}
805 echo "dir ${outputDir}" >> ${doneFile}
806 if summarizeLogs >> ${doneFile}; then
807 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
808 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
809 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
810 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
811 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
812 fi
813
814 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
815 return 0
816}
817
818goMerge()
819{
820 #generic root merge using CPass1 merge.C script
821 inputList=${1}
822 outputFile=${2}
823 configFile=${3-"becnhmark.config"}
824 shift 3
825 if ! parseConfig ${configFile} "$@"; then return 1; fi
826
827 #record the working directory provided by the batch system
828 batchWorkingDirectory=${PWD}
829
830 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
831 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
832 umask 0002
833 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
834 rm -f ${outputFile}
835 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
836 return 0
837}
838
839goSubmitMakeflow()
840{
841 #run
842 productionID=${1}
843 inputFileList=${2}
844 configFile=${3}
845 shift 3
846 extraOpts=("$@")
847 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
848
849 #record the working directory provided by the batch system
850 batchWorkingDirectory=${PWD}
851
852 [[ -z ${configFile} ]] && configFile="benchmark.config"
853 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
854
855 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
856 echo "setting the makflow path from the config: "
857 echo " export PATH=${makeflowPath}:${PATH}"
858 export PATH=${makeflowPath}:${PATH}
859 fi
860
861 #create the common output dir and the meta dir
862 commonOutputPath=${baseOutputDirectory}/${productionID}
863 if [[ -d ${commonOutputPath} ]]; then
864 echo "output dir ${commonOutputPath} exists!"
865 return 1
866 else
867 mkdir -p ${commonOutputPath}
868 fi
869 mkdir -p ${commonOutputPath}/meta
870
871 self=$(readlink -f "${0}")
872
873 cp ${self} ${commonOutputPath}
874 cp ${configFile} ${commonOutputPath}
875 cp ${inputList} ${commonOutputPath}
876
877 #submit - use makeflow if available, fall back to old stuff when makeflow not there
878 if which makeflow; then
879
880 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
881
882 makeflow ${makeflowOptions} benchmark.makeflow
883 cd ../
884 else
885 echo "no makeflow!"
886 fi
887 return 0
888}
889
890goGenerateMakeflow()
891{
892 #generate the makeflow file
893 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
894 productionID=${1}
895 inputFileList=${2}
896 configFile=${3}
897 shift 3
898 extraOpts=("$@")
899 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
900
901 #extra safety
902 if [[ -z ${commonOutputPath} ]]; then
903 commonOutputPath=${baseOutputDirectory}/${productionID}
904 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
905 fi
906
907 #record the working directory provided by the batch system
908 batchWorkingDirectory=${PWD}
909
910 [[ -z ${configFile} ]] && configFile="benchmark.config"
911 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
912
913 #these files will be made a dependency - will be copied to the working dir of the jobs
914 declare -a copyFiles
915 inputFiles=(
916 "OCDB.root"
917 "localOCDBaccessConfig.C"
918 "QAtrain_duo.C"
919 "runCPass1.sh"
920 "recCPass1.C"
921 "recCPass1_OuterDet.C"
922 "runCalibTrain.C"
923 "runCPass0.sh"
924 "recCPass0.C"
925 )
926 for file in ${inputFiles[*]}; do
927 [[ -f ${file} ]] && copyFiles+=("${file}")
928 done
929
930 #create the makeflow file
931 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
932 declare -A arr_cpass0_merged arr_cpass1_merged
933 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
934 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
935 declare -A arr_cpass0_profiled_outputs
936 declare -A listOfRuns
937 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
938 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
939 for runNumber in "${!listOfRuns[@]}"; do
940 [[ -z ${runNumber} ]] && continue
941 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
942
943 unset arr_cpass0_outputs
944 unset arr_cpass1_outputs
945 declare -a arr_cpass0_outputs
946 declare -a arr_cpass1_outputs
947
948 jobindex=0
949 inputFile=""
950 while read inputFile; do
951 currentDefaultOCDB=${defaultOCDB}
952 [[ -z ${autoOCDB} ]] && autoOCDB=1
953 if [[ ${autoOCDB} -ne 0 ]]; then
954 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
955 fi
956
957 #CPass0
958 arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
959 echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
960 echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
961 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
962 echo
963
964 #CPass1
965 arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
966 echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
967 echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
968 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
969 echo
970 ((jobindex++))
971
972 done< <(grep "/000${runNumber}/" ${inputFileList})
973
974 #CPass0 list of Calib files to merge
975 arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
976 echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
977 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
978 echo
979
980 #CPass0 merging
981 arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
982 echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
983 echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
984 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
985 echo
986
987 #CPass1 list of Calib/QA/ESD/filtered files
988 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
989 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
990 # this corresponds to the last merging stage)
991 arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
992 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
993 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
994 echo
995 arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
996 echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
997 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
998 echo
999 arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1000 echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1001 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1002 echo
1003 arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1004 echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1005 echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1006 echo
1007
1008 #CPass1 merging
1009 arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1010 echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1011 echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
1012 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1013 echo
1014
1015 #CPass0 wrapped in a profiling tool (valgrind,....)
1016 if [[ -n ${profilingCommand} ]]; then
1017 arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
1018 echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1019 echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
1020 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
1021 echo "\"useProfilingCommand=${profilingCommand}\""
1022 echo
1023 fi
1024
1025 done #runs
1026
1027 #Summary
1028 echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1029 echo -n " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1030 for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1031 echo
1032
1033 return 0
1034}
1035
1036goPrintValues()
1037{
1038 #print the values given the key from any number of files (store in output file on second argument)
1039 if [[ $# -lt 3 ]]; then
1040 echo "goPrintValues key outputFile inputFiles"
1041 echo "if outputFile is \"-\" dont redirect to a file"
1042 return
1043 fi
1044 key=${1}
1045 outputFile=${2}
1046 [[ ${outputFile} =~ "-" ]] && outputFile=""
1047 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1048 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1049 return 0
1050}
1051
1052goCreateQAplots()
1053{
1054 umask 0002
1055 mergedQAfileList=${1}
1056 productionID=${2}
1057 outputDir=${3}
1058 configFile=${4}
1059 shift 4
1060 if ! parseConfig ${configFile} ${@}; then return 1; fi
1061
1062 #record the working directory provided by the batch system
1063 batchWorkingDirectory=${PWD}
1064
1065 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1066
1067 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1068 [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1069 echo "${0} $*"
1070
1071 olddir=${PWD}
1072 mkdir -p ${outputDir}
1073 cd ${outputDir}
1074 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1075
1076 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1077
1078 cd ${olddir}
1079 return 0
1080}
1081
1082goTest()
1083{
1084 umask 0002
1085 exec 2>&1
1086 exec > >(tee test.log)
1087 echo "$@"
1088 echo something
1089 return 0
1090}
1091
1092alirootInfo()
1093{
1094 umask 0002
1095 # save aliroot repository info
1096 [[ -z "${ALICE_ROOT}" ]] && return 1
1097
1098 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1099 echo "\${ROOTSYS}=${ROOTSYS}"
1100 echo "\${PATH}=${PATH}"
1101 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1102
1103 pushd ${PWD}
1104 cd ${ALICE_ROOT}
1105 echo
1106
1107 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1108 git status
1109 echo ""
1110 echo ""
1111 git diff ${currentBranch}
1112 popd
1113 return 0
1114}
1115
1116setYear()
1117{
1118 #set the year
1119 # ${1} - year to be set
1120 # ${2} - where to set the year
1121 year1=$(guessYear ${1})
1122 year2=$(guessYear ${2})
1123 local path=${2}
1124 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1125 echo ${path}
1126 return 0
1127}
1128
1129guessPeriod()
1130{
1131 #guess the period from the path, pick the rightmost one
1132 local IFS="/"
1133 declare -a path=( ${1} )
1134 local dirDepth=${#path[*]}
1135 for ((x=${dirDepth}-1;x>=0;x--)); do
1136 local field=${path[${x}]}
1137 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1138 done
1139 echo ${period}
1140 return 0
1141}
1142
1143guessYear()
1144{
1145 #guess the year from the path, pick the rightmost one
1146 local IFS="/"
1147 declare -a path=( ${1} )
1148 local dirDepth=${#path[*]}
1149 for ((x=${dirDepth}-1;x>=0;x--)); do
1150 local field=${path[${x}]}
1151 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1152 done
1153 echo ${year}
1154 return 0
1155}
1156
1157guessRunNumber()
1158{
1159 #guess the run number from the path, pick the rightmost one
1160 #works for /path/foo/000123456/bar/...
1161 #and /path/foo.run123456.bar
1162 local IFS="/."
1163 declare -a path=( ${1} )
1164 local dirDepth=${#path[*]}
1165 for ((x=${dirDepth}-1;x>=0;x--)); do
1166 local field=${path[${x}]}
1167 field=${field/run/000}
1168 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1169 done
1170 echo ${runNumber}
1171 return 0
1172}
1173
1174validateLog()
1175{
1176 log=${1}
1177 errorConditions=(
1178 'There was a crash'
1179 'floating'
1180 'error while loading shared libraries'
1181 'std::bad_alloc'
1182 's_err_syswatch_'
1183 'Thread [0-9]* (Thread'
1184 'AliFatal'
1185 'core dumped'
1186 '\.C.*error:.*\.h: No such file'
1187 'line.*Aborted'
1188 )
1189
1190 warningConditions=(
1191 'This is serious !'
1192 'rocVoltage out of range:'
1193 )
1194
1195 local logstatus=0
1196 local errorSummary=""
1197 local warningSummary=""
1198
1199 for ((i=0; i<${#errorConditions[@]};i++)); do
1200 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1201 [[ -n ${tmp} ]] && tmp+=" : "
1202 errorSummary+=${tmp}
1203 done
1204
1205 for ((i=0; i<${#warningConditions[@]};i++)); do
1206 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1207 [[ -n ${tmp} ]] && tmp+=" : "
1208 warningSummary+=${tmp}
1209 done
1210
1211 if [[ -n ${errorSummary} ]]; then
1212 echo "${errorSummary}"
1213 return 1
1214 fi
1215
1216 if [[ -n ${warningSummary} ]]; then
1217 echo "${warningSummary}"
1218 return 2
1219 fi
1220
1221 return 0
1222}
1223
1224summarizeLogs()
1225{
1226 #print a summary of logs
1227 logFiles=(
1228 "*.log"
1229 "stdout"
1230 "stderr"
1231 )
1232
1233 #check logs
1234 local logstatus=0
1235 for log in ${logFiles[*]}; do
1236 finallog=${PWD%/}/${log}
1237 [[ ! -f ${log} ]] && continue
1238 errorSummary=$(validateLog ${log})
1239 validationStatus=$?
1240 if [[ ${validationStatus} -eq 0 ]]; then
1241 #in pretend mode randomly report an error in rec.log some cases
1242 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1243 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1244 echo "${finallog} OK"
1245 else
1246 echo "${finallog} OK"
1247 fi
1248 elif [[ ${validationStatus} -eq 1 ]]; then
1249 echo "${finallog} BAD ${errorSummary}"
1250 logstatus=1
1251 elif [[ ${validationStatus} -eq 2 ]]; then
1252 echo "${finallog} OK MWAH ${errorSummary}"
1253 fi
1254 done
1255
1256 #report core files
1257 while read x; do
1258 echo ${x}
1259 chmod 644 ${x}
1260 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1261 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1262
1263 return ${logstatus}
1264}
1265
1266spitOutLocalOCDBaccessConfig()
1267{
1268 umask 0002
1269 #find ${1} -name "*root" | \
1270 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1271 while read line
1272 do
1273 local tmp=${line#${1}}
1274 echo ${tmp%/*} | \
1275 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1276 done
1277 return 0
1278}
1279
1280goMakeLocalOCDBaccessConfig()
1281{
1282 umask 0002
1283 # make a script that sets the specific storages form all the root files produced by CPass0
1284 local localOCDBpathCPass0=${1}
1285 local OCDBpathPrefix=${2}
1286 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1287
1288 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1289 tar xzf ${localOCDBpathCPass0}
1290 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1291 fi
1292
1293 echo
1294 echo creating the specific storage script
1295 echo localOCDBaccessConfig.C
1296 echo based on OCDB: ${localOCDBaccessConfig}
1297 echo
1298
1299 local tempLocalOCDB=""
1300 if [[ -f localOCDBaccessConfig.C ]]; then
1301 tempLocalOCDB=$(mktemp)
1302 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1303 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1304 fi
1305
1306 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1307 echo "{" >> localOCDBaccessConfig.C
1308 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
1309 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1310 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
1311 echo "}" >> localOCDBaccessConfig.C
1312
1313 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1314
1315 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1316 echo
1317 echo "!!!!!!! CPass0 produced no OCDB entries"
1318 return 1
1319 fi
1320 return 0
1321}
1322
1323goMakeFilteredTrees()
1324{
1325 outputDir=${1}
1326 runNumber=${2}
1327 #get path to input list
1328 inputListfiles=${3}
1329 #get scale number for tracks
1330 filterT=${4}
1331 #get scale number for V0s
1332 filterV=${5}
1333 #get OCDB path (optional)
1334 OCDBpath=${6}
1335 #get max number of files
1336 maxFiles=${7-"1000000"}
1337 #get offset of first file
1338 offsetFile=${8-"0"}
1339 #get max number of events
1340 maxEvents=${9-"30000000"}
1341 #get offset of first event
1342 offsetEvent=${10-"0"}
1343 configFile=${11-"benchmark.config"}
1344 esdFileName=${12-"AliESDs_Barrel.root"}
1345 shift 12
1346 if ! parseConfig ${configFile} "$@"; then return 1; fi
1347
1348 #record the working directory provided by the batch system
1349 batchWorkingDirectory=${PWD}
1350
1351
1352 commonOutputPath=${PWD}
1353 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1354
1355 runpath=${outputDir}
1356 [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1357 [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1358
1359
1360 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1361
1362 mkdir -p ${outputDir}
1363 mkdir -p ${runpath}
1364
1365 cd ${runpath}
1366 [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
1367
1368 cat > filtering.log << EOF
1369 goMakeFilteredTrees config:
1370 runpath=${runpath}
1371 outputDir=${outputDir}
1372 commonOutputPath=${commonOutputPath}
1373 ALICE_ROOT=${ALICE_ROOT}
1374 PATH=${PATH}
1375 offsetEvent=$offsetEvent
1376 configFile=$configFile
1377 esdFileName=$esdFileName
1378EOF
1379
1380 if [[ -z ${pretend} ]];then
1381 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1382 else
1383 touch filtering.log FilterEvents_Trees.root
1384 fi
1385 pwd
1386 /bin/ls
1387 touch ${doneFile}
1388 summarizeLogs >> ${doneFile}
1389 echo mv -f * ${outputDir}
1390 mv -f * ${outputDir}
1391 [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1392 cd ${commonOutputPath}
1393 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1394 return 0
1395}
1396
1397submit()
1398{
1399 umask 0002
1400 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1401 JobID=${1}
1402 startID=${2}
1403 endID=${3}
1404 waitForJOBID=${4}
1405 command=${5}
1406 shift 5
1407 commandArgs=("$@")
1408
1409 #add quote strings around the extra arguments
1410 for ((i=0; i<${#commandArgs[@]}; i++)); do
1411 commandArgs[i]=\"${commandArgs[i]}\"
1412 done
1413
1414 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1415
1416 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1417
1418 batchSystem="SGE"
1419
1420 if [[ -z "${newFarm}" ]]
1421 then
1422 #old LSF
1423 # submit it (as job array)
1424 nFiles=$(( ${endID}-${startID}+1 ))
1425 while [ ${startID} -le ${nFiles} ] ; do
1426 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1427 endID=$(expr ${startID} + 999)
1428 else
1429 endID=${nFiles}
1430 fi
1431 if [[ ${waitForJOBID} -eq 0 ]]; then
1432 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1433 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1434 else
1435 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1436 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1437 fi
1438 startID=$(expr ${endID} + 1)
1439 done
1440 else
1441 #new SGE farm
1442 if [[ ${waitForJOBID} =~ "000" ]]; then
1443 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1444 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1445 else
1446 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1447 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1448 fi
1449 fi
1450 return 0
1451}
1452
1453goSubmitBatch()
1454{
1455 if [[ $# -lt 3 ]]; then
1456 echo "minimal use:"
1457 echo " ${0} submit fileList productionID configFile"
1458 return 0
1459 fi
1460
1461 productionID=${1}
1462 inputList=${2}
1463 configFile=${3:-"benchmark.config"}
1464 configFile=$(readlink -f ${configFile})
1465 shift 3
1466 extraOpts=("$@")
1467 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1468
1469 #record the working directory provided by the batch system
1470 batchWorkingDirectory=${PWD}
1471
1472 #redirect all output to submit.log
1473 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1474 exec 7>&1
1475 exec 1>submit_${productionID//"/"/_}.log 2>&1
1476
1477 umask 0002
1478 echo ${0}" submit $*"
1479 if [[ -z "${inputList}" || -z "${productionID}" ]]
1480 then
1481 echo
1482 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1483 echo
1484 return
1485 fi
1486
1487 # check if config file is there
1488 if [ ! -f ${configFile} ]; then
1489 echo "ERROR! Config File '${configFile}' not found" >&2
1490 return
1491 else
1492 echo "Using Config File: '${configFile}'"
1493 fi
1494
1495 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1496
1497 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1498 self=$(readlink -f "${0}")
1499 configPath=$(dirname ${configFile})
1500 export commonOutputPath=${baseOutputDirectory}/${productionID}
1501
1502 mkdir -p ${commonOutputPath}
1503 mkdir -p ${commonOutputPath}/logs
1504 mkdir -p ${commonOutputPath}/meta
1505
1506 cp ${self} ${commonOutputPath}
1507 cp ${configFile} ${commonOutputPath}
1508 cp ${inputList} ${commonOutputPath}
1509 self=${commonOutputPath}/${self##*/}
1510 chmod u+x ${self}
1511 configFile=${commonOutputPath}/${configFile##*/}
1512 inputList=${commonOutputPath}/${inputList##*/}
1513
1514 #convert to absolut pathnames
1515 inputList=$(readlink -f "${inputList}")
1516 #make list of runs
1517 if [[ -z ${runNumber} ]]; then
1518 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1519 else
1520 listOfRuns=${runNumber}
1521 fi
1522
1523 alirootSource=$(readlink -f "${alirootSource}")
1524
1525 echo ""
1526 echo "### BEGIN CONFIGURATION ###"
1527 echo ""
1528 echo "GENERAL:"
1529 echo ""
1530 echo " productionID: ${productionID}"
1531 echo " batchCommand: ${batchCommand}"
1532 echo " batchFlags: ${batchFlags}"
1533 echo " alirootEnv: ${alirootEnv}"
1534 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1535 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
1536 echo " inputList: ${inputList}"
1537 echo " configPath: ${configPath}"
1538 echo " commonOutputPath: ${commonOutputPath}"
1539 echo " defaultOCDB: ${defaultOCDB}"
1540 echo " autoOCDB: ${autoOCDB}"
1541 echo " recoTriggerOptions: ${recoTriggerOptions}"
1542 echo " runs:"
1543 echo " ${listOfRuns[*]}"
1544 echo ""
1545 echo "THE TRAIN WILL RUN:"
1546
1547 if [ ${runCPass0reco} -eq 1 ]; then
1548 echo " Pass0 - Recontruction"
1549 fi
1550
1551 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1552 echo " Pass0 - merging and OCDB export"
1553 fi
1554
1555 if [ ${runCPass1reco} -eq 1 ]; then
1556 echo " Pass1 - Recontruction"
1557 fi
1558 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1559 echo " Pass1 - merging and OCDB export"
1560 fi
1561
1562 echo ""
1563 echo "LIMITS:"
1564 echo " max. Events/Chunk: ${nEvents}"
1565 echo " max. Number of Chunks per Run: ${nMaxChunks}"
1566 echo ""
1567 echo "### END CONFIGURATION ###"
1568 echo ""
1569
1570
1571 # check if input file is there
1572 if [ ! -f ${inputList} ]; then
1573 echo "ERROR! Input List '${inputList}' not found" >&2
1574 return
1575 fi
1576
1577 # define jobid (for dependent jobs)
1578 date=$(date +%Y_%m_%d_%H%M%S)
1579 #for each run we submit one jobarray:
1580 for runNumber in ${listOfRuns[*]}; do
1581
1582 [[ -z ${runNumber} ]] && continue
1583 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1584
1585 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1586 JOBID1="p0_${JOBpostfix}"
1587 JOBID1wait="w0_${JOBpostfix}"
1588 JOBID2="m0_${JOBpostfix}"
1589 JOBID2wait="wm0_${JOBpostfix}"
1590 JOBID3="op0_${JOBpostfix}"
1591 JOBID3wait="wop0_${JOBpostfix}"
1592 JOBID4="p1_${JOBpostfix}"
1593 JOBID4wait="w1_${JOBpostfix}"
1594 JOBID5="m1_${JOBpostfix}"
1595 JOBID5wait="wm1_${JOBpostfix}"
1596 JOBID6="s1_${JOBpostfix}"
1597 JOBID6wait="ws1_${JOBpostfix}"
1598 JOBID7="QA_${JOBpostfix}"
1599 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1600 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1601 LASTJOB="000"
1602
1603 oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1604
1605 currentDefaultOCDB=${defaultOCDB}
1606 [[ -z ${autoOCDB} ]] && autoOCDB=1
1607 if [[ ${autoOCDB} -ne 0 ]]; then
1608 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1609 fi
1610
1611 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1612
1613 ###############################################################################
1614 #run one chunk with valgrind:
1615 if [[ -n ${profilingCommand} ]]; then
1616 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1617 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1618 submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1619 fi
1620
1621 ################################################################################
1622 ################################################################################
1623 # run the CPass0 if requested
1624
1625 if [ ${runCPass0reco} -eq 1 ]; then
1626
1627 echo
1628 echo "starting CPass0... for run ${runNumber}"
1629 echo
1630
1631 # create directory and copy all files that are needed
1632 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1633 mkdir -p ${targetDirectory}
1634
1635 filesCPass0=(
1636 "${configPath}/runCPass0.sh"
1637 "${configPath}/recCPass0.C"
1638 "${configPath}/runCalibTrain.C"
1639 "${configPath}/localOCDBaccessConfig.C"
1640 "${configPath}/OCDB*.root"
1641 "${configPath}/sim.C"
1642 "${configPath}/rec.C"
1643 "${configPath}/Config.C"
1644 )
1645 for file in ${filesCPass0[*]}; do
1646 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1647 done
1648
1649 localInputList=${targetDirectory}/${inputList##*/}
1650 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1651 # limit nFiles to nMaxChunks
1652 nFiles=$(wc -l < ${localInputList})
1653 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1654 echo "raw files in list: ${nFiles}"
1655 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1656 nFiles=${nMaxChunks}
1657 fi
1658 echo "raw files to process: ${nFiles}"
1659 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1660 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1661 nFilesToWaitFor=${nFiles}
1662 else
1663 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1664 fi
1665 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1666 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1667
1668 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1669
1670 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1671 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1672 LASTJOB=${JOBID1wait}
1673
1674 fi #end running CPass0
1675 ################################################################################
1676
1677
1678 ################################################################################
1679 # submit merging of CPass0, depends on the reconstruction
1680
1681 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1682
1683 echo
1684 echo "submit CPass0 merging for run ${runNumber}"
1685 echo
1686
1687 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1688 mkdir -p ${targetDirectory}
1689
1690 #copy the scripts
1691 filesMergeCPass0=(
1692 "${configPath}/OCDB.root"
1693 "${configPath}/mergeMakeOCDB.byComponent.sh"
1694 "${configPath}/mergeMakeOCDB.sh"
1695 "${configPath}/localOCDBaccessConfig.C"
1696 "${configPath}/mergeByComponent.C"
1697 "${configPath}/makeOCDB.C"
1698 "${configPath}/merge.C"
1699 )
1700 for file in ${filesMergeCPass0[*]}; do
1701 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1702 done
1703
1704 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1705 LASTJOB=${JOBID2}
1706
1707 if [[ -n ${generateMC} ]]; then
1708 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1709 LASTJOB="mrl${JOBpostfix}"
1710 fi
1711
1712 echo
1713 fi
1714 # end of merging CPass0
1715 ################################################################################
1716
1717 ################################################################################
1718 ################################################################################
1719 # run the CPass1 if requested
1720
1721 if [ ${runCPass1reco} -eq 1 ]; then
1722
1723 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1724 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1725
1726 # safety feature: if we are re-running for any reason we want to delete the previous output first.
1727 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1728
1729 echo
1730 echo "starting CPass1... for run ${runNumber}"
1731 echo
1732
1733 # create directory and copy all files that are needed
1734 mkdir -p ${targetDirectory}
1735
1736 filesCPass1=(
1737 "${configPath}/runCPass1.sh"
1738 "${configPath}/recCPass1.C"
1739 "${configPath}/recCPass1_OuterDet.C"
1740 "${configPath}/runCalibTrain.C"
1741 "${configPath}/QAtrain_duo.C"
1742 "${configPath}/localOCDBaccessConfig.C"
1743 "${configPath}/OCDB.root"
1744 )
1745 for file in ${filesCPass1[*]}; do
1746 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1747 done
1748
1749 if [[ -n ${generateMC} ]]; then
1750 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1751 else
1752 localInputList=${targetDirectory}/${inputList##*/}
1753 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1754 fi
1755 # limit nFiles to nMaxChunks
1756 nFiles=$(wc -l < ${localInputList})
1757 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1758 echo "raw files in list: ${nFiles}"
1759 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1760 nFiles=${nMaxChunks}
1761 fi
1762 echo "raw files to process: ${nFiles}"
1763 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1764 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1765 nFilesToWaitFor=${nFiles}
1766 else
1767 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1768 fi
1769 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1770 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1771
1772 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1773
1774 ################################################################################
1775 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1776 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1777 LASTJOB=${JOBID4wait}
1778 ################################################################################
1779
1780 echo
1781 fi #end running CPass1
1782
1783 ################################################################################
1784 # submit merging of CPass1, depends on the reconstruction
1785 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1786
1787 echo
1788 echo "submit CPass1 merging for run ${runNumber}"
1789 echo
1790
1791 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1792 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1793 mkdir -p ${targetDirectory}
1794
1795 # copy files
1796 filesMergeCPass1=(
1797 "${configPath}/OCDB.root"
1798 "${configPath}/localOCDBaccessConfig.C"
1799 "${configPath}/mergeMakeOCDB.byComponent.sh"
1800 "${configPath}/mergeByComponent.C"
1801 "${configPath}/makeOCDB.C"
1802 "${configPath}/merge.C"
1803 "${configPath}/mergeMakeOCDB.sh"
1804 "${configPath}/QAtrain_duo.C"
1805 )
1806 for file in ${filesMergeCPass1[*]}; do
1807 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1808 done
1809
1810 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1811 LASTJOB=${JOBID5}
1812 echo
1813 fi
1814
1815 ###############################
1816 #if [ ${runESDfiltering} -eq 1 ]; then
1817 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1818 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1819 # echo
1820 # echo submitting filtering for run ${runNumber}
1821 # echo
1822 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1823 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1824 # LASTJOB=${JOBfilterESDcpass1}
1825 #fi
1826
1827 done
1828
1829 #################################################################################
1830 #################################################################################
1831 #if [ ${runESDfiltering} -eq 1 ]; then
1832 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1833 #else
1834 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1835 #fi
1836 LASTJOB=${JOBID5wait}
1837
1838 #################################################################################
1839 echo
1840 echo "submit make a summary"
1841 echo
1842
1843 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1844 LASTJOB=${JOBID6}
1845 #################################################################################
1846
1847 #restore stdout
1848 exec 1>&7 7>&-
1849 echo "jobs submitted."
1850 return 0
1851}
1852
1853goWaitForOutput()
1854{
1855 umask 0002
1856 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1857 echo searchPath=${1}
1858 echo fileName=${2}
1859 echo numberOfFiles=${3}
1860 echo maxSecondsToWait=${4}
1861 searchPath=${1}
1862 fileName=${2}
1863 numberOfFiles=${3}
1864 maxSecondsToWait=${4}
1865 extraFindOptions=${5}
1866 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1867 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1868 while true; do
1869 #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1870 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1871 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1872 [[ ${n} -ge ${numberOfFiles} ]] && break
1873 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1874 sleep 60
1875 done
1876 echo "DONE! exiting..."
1877 return 0
1878}
1879
1880mergeSysLogs()
1881{
1882 outputFile=${1}
1883 shift
1884 inputFiles="$@"
1885 i=0
1886 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1887 while read x; do
1888 runNumber=$(guessRunNumber ${x})
1889 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1890 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1891 (( i++ ))
1892 done < <(ls -1 ${inputFiles}) > ${outputFile}
1893 return 0
1894}
1895
1896goMakeMergedSummaryTree()
1897{
1898 # create list of calibration entries
1899 # takes no arguments, just run it in the base output
1900 # directory with the following files in the working directory
1901 #
1902 # Calibration file lists:
1903 # cpass0.dcsTree.list, cpass1.dcsTree.list
1904 # QA trending root file:
1905 # trending.root
1906 #
1907 # Production infoo ascii files:
1908 # summary_pass0.tree
1909 # summary_pass1.tree
1910 #
1911
1912 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1913 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1914 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1915 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1916 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1917
1918 #first, dump the C macro to file
1919 cat << EOF > mergeTree.C
1920 //
1921 // Merge summary information
1922 // Following files are expected to be in the working directory
1923 //
1924 // Calibration file lists:
1925 // cpass0.dcsTree.list, cpass1.dcsTree.list
1926 // QA trending root file:
1927 // trending.root
1928 //
1929 // Production infoo ascii files:
1930 // summary_pass0.tree
1931 // summary_pass1.tree
1932 //
1933 void mergeTree(){
1934 //
1935 //
1936 //
1937 // Calibration values dump
1938 //
1939 //Printf("MakeTreeFromList cpass0.dcsTree.list");
1940 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1941 //Printf("MakeTreeFromList cpass1.dcsTree.list");
1942 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1943 //
1944 // Calibration status dump
1945 //
1946 TFile *fprod = TFile::Open("fproduction.root","recreate");
1947 TTree tree0, tree1;
1948 //Printf("reading summary_pass0.tree");
1949 tree0.ReadFile("summary_pass0.tree");
1950 //Printf("reading summary_pass1.tree");
1951 tree1.ReadFile("summary_pass1.tree");
1952 tree0.Write("CPass0");
1953 tree1.Write("CPass1");
1954 fprod->Close();
1955 //
1956 //
1957 //
1958 TString stringSetup="";
1959 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
1960 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
1961 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
1962 //
1963 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
1964 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
1965 //
1966 //Printf("stringSetup: %s", stringSetup.Data());
1967 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1968 }
1969EOF
1970
1971 aliroot -b -q "mergeTree.C" > mergeTrees.log
1972 return $?
1973}
1974
1975goMakeSummary()
1976{
1977 #all the final stuff goes in here for ease of use:
1978 # summary logs
1979 # qa plot making
1980 # final file lists
1981 #some defaults:
1982 log="summary.log"
1983 productionID="qa"
1984
1985 configFile=${1}
1986 shift 1
1987 extraOpts=("$@")
1988 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1989
1990 configFile=$(readlink -f ${configFile})
1991
1992 #record the working directory provided by the batch system
1993 batchWorkingDirectory=${PWD}
1994
1995 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1996
1997 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1998
1999 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2000
2001 #copy some useful stuff
2002 #and go to the commonOutputPath
2003 cp ${configFile} ${commonOutputPath}
2004 cd ${commonOutputPath}
2005
2006 exec &> >(tee ${log})
2007
2008 #summarize the global stuff
2009 echo "env script: ${alirootSource} ${alirootEnv}"
2010 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2011 echo "commonOutputPath=${commonOutputPath}"
2012
2013 #summarize the stacktraces
2014 awk '
2015 BEGIN {
2016 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2017 RS="#[0-9]*";
2018 aliroot=0;
2019 }
2020 {
2021 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2022 gsub("#","",RT);
2023 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2024 }
2025 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2026 awk '
2027 BEGIN {
2028 RS="#[0-9]*";
2029 aliroot=0;
2030 }
2031 {
2032 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2033 gsub("#","",RT);
2034 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2035 }
2036 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2037
2038 echo total numbers for the production:
2039 echo
2040 awk 'BEGIN {nFiles=0;nCore=0;}
2041 /^calibfile/ {nFiles++;}
2042 /core dumped/ {nCore++i;}
2043 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2044 awk 'BEGIN {nOK=0; nBAD=0; }
2045 /\/rec.log OK/ {nOK++;}
2046 /\/rec.log BAD/ {nBAD++;}
2047 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2048 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2049 awk 'BEGIN {nOK=0; nBAD=0; }
2050 /\/calib.log OK/ {nOK++;}
2051 /\/calib.log BAD/ {nBAD++;}
2052 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2053
2054 awk 'BEGIN {nOK=0; nBAD=0; }
2055 /merge.log OK/ {nOK++;}
2056 /merge.log BAD/ {nBAD++;}
2057 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2058 awk 'BEGIN {nOK=0; nBAD=0; }
2059 /ocdb.log OK/ {nOK++;}
2060 /ocdb.log BAD/ {nBAD++;}
2061 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2062
2063 echo
2064 awk 'BEGIN {nFiles=0;nCore=0;}
2065 /^calibfile/ {nFiles++;}
2066 /core dumped/ {nCore++;}
2067 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2068 awk 'BEGIN {nOK=0; nBAD=0; }
2069 /\/rec.log OK/ {nOK++;}
2070 /\/rec.log BAD/ {nBAD++;}
2071 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2072 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2073 awk 'BEGIN {nOK=0; nBAD=0; }
2074 /\/calib.log OK/ {nOK++;}
2075 /\/calib.log BAD/ {nBAD++;}
2076 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2077
2078 awk 'BEGIN {nOK=0; nBAD=0; }
2079 /merge.log OK/ {nOK++;}
2080 /merge.log BAD/ {nBAD++;}
2081 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2082 awk 'BEGIN {nOK=0; nBAD=0; }
2083 /ocdb.log OK/ {nOK++;}
2084 /ocdb.log BAD/ {nBAD++;}
2085 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2086
2087 echo
2088 echo per run stats:
2089 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2090do
2091 dir=$(goPrintValues calibfile - ${x})
2092 runNumber=$(guessRunNumber ${dir})
2093 [[ -z ${runNumber} ]] && continue
2094
2095 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2096 statusCPass0=( $(
2097 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2098 /\/rec.log OK/ {nOKrec++;}
2099 /\/rec.log BAD/ {nBADrec++;}
2100 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2101 /stderr OK/ {nOKstderr++;}
2102 /\/calib.log OK/ {nOKcalib++;}
2103 /\/calib.log BAD/ {nBADcalib++}
2104 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2105 ) )
2106 fi
2107
2108 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2109 statusCPass1=( $(
2110 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2111 /\/rec.log OK/ {nOKrec++;}
2112 /\/rec.log BAD/ {nBADrec++;}
2113 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2114 /stderr OK/ {nOKstderr++;}
2115 /\/calib.log OK/ {nOKcalib++;}
2116 /\/calib.log BAD/ {nBADcalib++}
2117 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2118 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2119 /\/qa_outer.log OK/ {nQAouterOK++;}
2120 /\/qa_outer.log BAD/ {nQAouterBAD++;}
2121 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2122 ) )
2123 fi
2124
2125 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2126 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2127 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2128
2129 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2130done
2131
2132 #make lists with output files - QA, trending, filtering and calibration
2133 ### wait for the merging of all runs to be over ###
2134 rm -f qa.list
2135 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2136 rm -f calib.list
2137 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2138 rm -f trending.list
2139 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2140 rm -f filtering.list
2141 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2142 rm -f cpass0.dcsTree.list
2143 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2144 rm -f cpass1.dcsTree.list
2145 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2146
2147 #merge trending
2148 rm -f ${commonOutputPath}/trending_merged.root
2149 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2150
2151 goMakeSummaryTree ${commonOutputPath} 0
2152 goMakeSummaryTree ${commonOutputPath} 1
2153
2154 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2155
2156 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2157 echo goMakeMergedSummaryTree PWD=$PWD
2158 goMakeMergedSummaryTree
2159
2160 #if set, email the summary
2161 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2162
2163 return 0
2164}
2165
2166goMakeSummaryTree()
2167{
2168 if [[ $# -lt 1 ]] ; then
2169 return
2170 fi
2171 #1. define vars/arrays
2172 DIR=${1} #use input or exec in current dir
2173 pass=${2-"0"} #pass from input
2174 outfile="summary_pass${pass}.tree"
2175 Ncolumns=0
2176 test -f ${outfile} && : >${outfile}
2177 errfile=${outfile/tree/err}
2178 test -f ${errfile} && : >${errfile}
2179
2180 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2181 Ncounter=${#counterString[@]}
2182
2183 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2184 Nstatus=${#statusString[@]}
2185
2186
2187 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2188 Nrates=${#ratesString[@]}
2189
2190 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2191 Nruns=${#runs[@]}
2192
2193 echo -n runnumber/I >>${outfile}
2194 echo -n :cpass${pass}status/I >>${outfile}
2195 echo -n :cpass${pass}QAstatus/I >>${outfile}
2196 for i in ${ratesString[@]}; do
2197 echo -n :${i}OK/I >>${outfile}
2198 echo -n :${i}BAD/I >>${outfile}
2199
2200 done
2201 for i in ${counterString[@]} ${statusString[@]} ; do
2202 echo -n :${i}/I >>${outfile}
2203 done
2204 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2205 echo >> ${outfile}
2206
2207 #2. loop runs
2208
2209 for runnumber in ${runs[@]} ; do
2210
2211
2212
2213 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2214 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2215 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2216 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2217 echo "${filemerge} does not exist!" >>${errfile}
2218 continue
2219 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2220 echo "${filejob} does not exist!" >>${errfile}
2221 echo -n ${runnumber} >> ${outfile}
2222 for i in $(seq ${Ncolumns}) ; do
2223 echo -n "-1" >> ${outfile}
2224 done
2225 echo >> ${outfile}
2226 continue
2227 fi
2228 echo -n ${runnumber} >> ${outfile}
2229 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2230 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2231 echo -n " ${passStatus}" >> ${outfile}
2232 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2233 echo -n " ${qaStatus}" >> ${outfile}
2234
2235
2236 #fill OK/BAD rates
2237 for i in $(seq 0 $((${Nrates}-1))) ; do
2238 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2239 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2240
2241 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2242 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2243 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2244 fi
2245 echo -n " ${var1}" >> ${outfile}
2246 echo -n " ${var2}" >> ${outfile}
2247 done
2248
2249 if [[ -f ${fileOCDB} ]] ; then
2250 #fill counter
2251 for i in $(seq 0 $((${Ncounter}-1))) ; do
2252 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2253 echo -n " ${var1:-"-1"}" >> ${outfile}
2254 done
2255
2256 #fill status
2257 for i in $(seq 0 $((${Nstatus}-1))) ; do
2258 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2259 echo -n " ${var1:-"-1"}" >> ${outfile}
2260 done
2261 fi
2262 echo >> ${outfile}
2263 done
2264
2265 return 0
2266}
2267
2268parseConfig()
2269{
2270 configFile=${1}
2271 shift
2272 args=("$@")
2273
2274 #some defaults
2275 #autoOCDB=0
2276 defaultOCDB="raw://"
2277 #runNumber=167123
2278 #makeflowPath="/hera/alice/aux/cctools/bin"
2279 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2280 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2281 makeflowOptions=""
2282 #batchCommand="/usr/bin/qsub"
2283 batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2284 baseOutputDirectory="$PWD/output"
2285 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2286 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2287 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2288 reconstructInTemporaryDir=0
2289 recoTriggerOptions="\"\""
2290 percentProcessedFilesToContinue=100
2291 maxSecondsToWait=$(( 3600*24 ))
2292 nEvents=-1
2293 nMaxChunks=0
2294 postSetUpActionCPass0=""
2295 postSetUpActionCPass1=""
2296 runCPass0reco=1
2297 runCPass0MergeMakeOCDB=1
2298 runCPass1reco=1
2299 runCPass1MergeMakeOCDB=1
2300 runESDfiltering=1
2301 filteringFactorHighPt=1e2
2302 filteringFactorV0s=1e1
2303 MAILTO=""
2304 #pretend=1
2305 #dontRedirectStdOutToLog=1
2306 logToFinalDestination=1
2307 ALIROOT_FORCE_COREDUMP=1
2308
2309 #first, source the config file
2310 if [ -f ${configFile} ]; then
2311 source ${configFile}
2312 else
2313 echo "config file ${configFile} not found!, skipping..."
2314 fi
2315
2316 unset encodedSpaces
2317 for opt in "${args[@]}"; do
2318 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && break
2319 done
2320
2321 #then, parse the options as they override the options from file
2322 for opt in "${args[@]}"; do
2323 if [[ ! "${opt}" =~ .*=.* ]]; then
2324 echo "badly formatted option ${var}, should be: option=value, stopping..."
2325 return 1
2326 fi
2327 local var="${opt%%=*}"
2328 local value="${opt#*=}"
2329 [[ -n ${encodedSpaces} ]] && value=$(decSpaces "${value}")
2330 echo "${var} = ${value}"
2331 export ${var}="${value}"
2332 done
2333
2334 #do some checking
2335 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2336
2337 #export the aliroot function if defined to override normal behaviour
2338 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2339
2340 return 0
2341}
2342
2343aliroot()
2344{
2345 args="$@"
2346 if [[ -n ${useProfilingCommand} ]]; then
2347 valgrindLogFile="cpu.txt"
2348 [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2349 [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2350 [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2351 echo running ${useProfilingCommand} aliroot ${args}
2352 ${useProfilingCommand} aliroot ${args}
2353 else
2354 #to prevent an infinite recursion use "command aliroot" to disable
2355 #aliases and functions
2356 echo running command aliroot ${args}
2357 command aliroot "$@"
2358 fi
2359 return 0
2360}
2361
2362guessRunData()
2363{
2364 #guess the period from the path, pick the rightmost one
2365 period=""
2366 runNumber=""
2367 year=""
2368 pass=""
2369 legoTrainRunNumber=""
2370 dataType=""
2371
2372 local shortRunNumber=""
2373 local IFS="/"
2374 declare -a path=( $1 )
2375 local dirDepth=$(( ${#path[*]}-1 ))
2376 i=0
2377 #for ((x=${dirDepth};x>=0;x--)); do
2378 for ((x=0;x<=${dirDepth};x++)); do
2379
2380 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2381 local field=${path[${x}]}
2382 local fieldNext=${path[$((x+1))]}
2383
2384 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2385 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2386 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2387 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2388 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2389 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2390 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2391 (( i++ ))
2392 done
2393 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2394 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2395
2396 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2397 if [[ -z ${runNumber}} ]];
2398 then
2399 #error condition
2400 return 1
2401 else
2402 #ALL OK
2403 return 0
2404 fi
2405 return 0
2406}
2407
2408#these functions encode strings to and from a space-less form
2409#use when spaces are not well handled (e.g. in arguments to
2410#commands in makeflow files, etc.
2411encSpaces()(a="${1//,/\\,}";echo "${a// /,}")
2412decSpaces()(a="${1//\\,/\\ }";b="${a//,/ }";echo "${b//\\ /,}")
2413
2414main "$@"